LLVM 24.0.0git
SIInsertWaitcnts.cpp
Go to the documentation of this file.
1//===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert wait instructions for memory reads and writes.
11///
12/// Memory reads and writes are issued asynchronously, so we need to insert
13/// S_WAITCNT instructions when we want to access any of their results or
14/// overwrite any register that's used asynchronously.
15///
16/// TODO: This pass currently keeps one timeline per hardware counter. A more
17/// finely-grained approach that keeps one timeline per event type could
18/// sometimes get away with generating weaker s_waitcnt instructions. For
19/// example, when both SMEM and LDS are in flight and we need to wait for
20/// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21/// but the pass will currently generate a conservative lgkmcnt(0) because
22/// multiple event types are in flight.
23//
24//===----------------------------------------------------------------------===//
25
26#include "AMDGPU.h"
27#include "AMDGPUHWEvents.h"
28#include "AMDGPUWaitcntUtils.h"
29#include "GCNSubtarget.h"
37#include "llvm/IR/Dominators.h"
41
42using namespace llvm;
43
45
46#define DEBUG_TYPE "si-insert-waitcnts"
47
48namespace {
49
50template <typename EmitWaitcntFn>
51static void EmitExpandedWaitcnt(unsigned Outstanding, unsigned Target,
52 EmitWaitcntFn &&EmitWaitcnt) {
53 // Emit waitcnts from (Outstanding - 1) down to Target.
54 for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
55 EmitWaitcnt(I);
56 EmitWaitcnt(Target);
57}
58
59/// Integer IDs used to track vector memory locations we may have to wait on.
60/// Encoded as u16 chunks:
61///
62/// [0, REGUNITS_END ): MCRegUnit
63/// [LDSDMA_BEGIN, LDSDMA_END ) : LDS DMA IDs
64///
65/// NOTE: The choice of encoding these as "u16 chunks" is arbitrary.
66/// It gives (2 << 16) - 1 entries per category which is more than enough
67/// for all register units. MCPhysReg is u16 so we don't even support >u16
68/// physical register numbers at this time, let alone >u16 register units.
69/// In any case, an assertion in "WaitcntBrackets" ensures REGUNITS_END
70/// is enough for all register units.
71using VMEMID = uint32_t;
72
73enum : VMEMID {
74 TRACKINGID_RANGE_LEN = (1 << 16),
75
76 // Important: MCRegUnits must always be tracked starting from 0, as we
77 // need to be able to convert between a MCRegUnit and a VMEMID freely.
78 REGUNITS_BEGIN = 0,
79 REGUNITS_END = REGUNITS_BEGIN + TRACKINGID_RANGE_LEN,
80
81 // Note for LDSDMA: LDSDMA_BEGIN corresponds to the "common"
82 // entry, which is updated for all LDS DMA operations encountered.
83 // Specific LDS DMA IDs start at LDSDMA_BEGIN + 1.
84 NUM_LDSDMA = TRACKINGID_RANGE_LEN,
85 LDSDMA_BEGIN = REGUNITS_END,
86 LDSDMA_END = LDSDMA_BEGIN + NUM_LDSDMA,
87};
88
89/// Convert a MCRegUnit to a VMEMID.
90static constexpr VMEMID toVMEMID(MCRegUnit RU) {
91 return static_cast<unsigned>(RU);
92}
93
94} // namespace
95
96namespace {
97
98// Maps values of InstCounterType to the instruction that waits on that
99// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
100// returns true, and does not cover VA_VDST or VM_VSRC.
101static const unsigned
102 instrsForExtendedCounterTypes[AMDGPU::NUM_EXTENDED_INST_CNTS] = {
103 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
104 AMDGPU::S_WAIT_EXPCNT, AMDGPU::S_WAIT_STORECNT,
105 AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
106 AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
107 AMDGPU::S_WAIT_ASYNCCNT, AMDGPU::S_WAIT_TENSORCNT};
108
109// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
110// code but still need to be processed by this pass for async vmcnt tracking.
111static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
112 switch (MI.getOpcode()) {
113 case AMDGPU::ASYNCMARK:
114 case AMDGPU::WAIT_ASYNCMARK:
115 return false;
116 default:
117 return MI.isMetaInstruction();
118 }
119}
120
121static bool updateVMCntOnly(const MachineInstr &Inst) {
122 return (SIInstrInfo::isVMEM(Inst) && !SIInstrInfo::isFLAT(Inst)) ||
124}
125
126#ifndef NDEBUG
127static bool isNormalMode(AMDGPU::InstCounterType MaxCounter) {
128 return MaxCounter == AMDGPU::NUM_NORMAL_INST_CNTS;
129}
130#endif // NDEBUG
131
132class WaitcntBrackets;
133
134// This abstracts the logic for generating and updating S_WAIT* instructions
135// away from the analysis that determines where they are needed. This was
136// done because the set of counters and instructions for waiting on them
137// underwent a major shift with gfx12, sufficiently so that having this
138// abstraction allows the main analysis logic to be simpler than it would
139// otherwise have had to become.
140class WaitcntGenerator {
141protected:
142 const GCNSubtarget &ST;
143 const SIInstrInfo &TII;
144 AMDGPU::IsaVersion IV;
145 AMDGPU::InstCounterType MaxCounter;
146 bool OptNone;
147 bool ExpandWaitcntProfiling = false;
148 const AMDGPU::HardwareLimits &Limits;
149
150 // Soft waitcnts whose erasures are postponed so we can confirm if they're
151 // truly redundant or if they're waiting on a loop-carried dependency.
152 DenseSet<MachineInstr *> DeferredErasures;
153
154public:
155 WaitcntGenerator() = delete;
156 WaitcntGenerator(const WaitcntGenerator &) = delete;
157 WaitcntGenerator(const MachineFunction &MF,
158 AMDGPU::InstCounterType MaxCounter,
159 const AMDGPU::HardwareLimits &Limits)
160 : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
161 IV(AMDGPU::getIsaVersion(ST.getCPU())), MaxCounter(MaxCounter),
162 OptNone(MF.getFunction().hasOptNone() ||
163 MF.getTarget().getOptLevel() == CodeGenOptLevel::None),
164 ExpandWaitcntProfiling(
165 MF.getFunction().hasFnAttribute("amdgpu-expand-waitcnt-profiling")),
166 Limits(Limits) {}
167
168 // Return true if the current function should be compiled with no
169 // optimization.
170 bool isOptNone() const { return OptNone; }
171
172 unsigned getLimit(AMDGPU::InstCounterType E) const { return Limits.get(E); }
173
174 // Edits an existing sequence of wait count instructions according
175 // to an incoming Waitcnt value, which is itself updated to reflect
176 // any new wait count instructions which may need to be generated by
177 // WaitcntGenerator::createNewWaitcnt(). It will return true if any edits
178 // were made.
179 //
180 // This editing will usually be merely updated operands, but it may also
181 // delete instructions if the incoming Wait value indicates they are not
182 // needed. It may also remove existing instructions for which a wait
183 // is needed if it can be determined that it is better to generate new
184 // instructions later, as can happen on gfx12.
185 virtual bool
186 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
187 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
189
190 bool eraseDeferredWaitcnts() {
191 bool Modified = false;
192 for (MachineInstr *MI : DeferredErasures) {
193 MI->eraseFromParent();
194 Modified = true;
195 }
196 DeferredErasures.clear();
197 return Modified;
198 }
199
200 // Transform a soft waitcnt into a normal one.
201 bool promoteSoftWaitCnt(MachineInstr *Waitcnt) const;
202
203 // Generates new wait count instructions according to the value of
204 // Wait, returning true if any new instructions were created.
205 // ScoreBrackets is used for profiling expansion.
206 virtual bool createNewWaitcnt(MachineBasicBlock &Block,
208 AMDGPU::Waitcnt Wait,
209 const WaitcntBrackets &ScoreBrackets) = 0;
210
211 // Returns the set of HWEvents that corresponds to counter \p T.
212 virtual HWEvents getWaitEvents(AMDGPU::InstCounterType T) const = 0;
213
214 /// \returns the counter that corresponds to event \p E.
215 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
216 assert(E.size() == 1 && "Cannot handle a mask of events!");
217 for (auto T : AMDGPU::inst_counter_types()) {
218 if (getWaitEvents(T) & E)
219 return T;
220 }
221 llvm_unreachable("event type has no associated counter");
222 }
223
224 // Returns a new waitcnt with all counters except VScnt set to 0. If
225 // IncludeVSCnt is true, VScnt is set to 0, otherwise it is set to ~0u.
226 // AsyncCnt and TensorCnt always default to ~0u (don't wait for it). They
227 // are only updated when a call to @llvm.amdgcn.wait.asyncmark() is
228 // processed.
229 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
230
231 virtual ~WaitcntGenerator() = default;
232};
233
234class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
235 static constexpr const HWEvents
236 WaitEventMaskForInstWithoutVscnt[AMDGPU::NUM_INST_CNTS] = {
237 // LOAD_CNT
238 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
239 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS |
240 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
241 // DS_CNT
242 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
243 HWEvents::SQ_MESSAGE,
244 // EXP_CNT
245 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
246 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
247 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
251
252 // Hardware with VScnt has non-empty STORE_CNT wait events.
253 static constexpr const HWEvents
254 WaitEventMaskForInstWithVscnt[AMDGPU::NUM_INST_CNTS] = {
255 // LOAD_CNT
256 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
257 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
258 // DS_CNT
259 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
260 HWEvents::SQ_MESSAGE,
261 // EXP_CNT
262 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
263 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
264 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
265 // STORE_CNT
266 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
270
271public:
272 using WaitcntGenerator::WaitcntGenerator;
273 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
274 MachineInstr &OldWaitcntInstr,
275 AMDGPU::Waitcnt &Wait,
277
278 bool createNewWaitcnt(MachineBasicBlock &Block,
280 AMDGPU::Waitcnt Wait,
281 const WaitcntBrackets &ScoreBrackets) override;
282
283 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
284 return ST.hasVscnt() ? WaitEventMaskForInstWithVscnt[T]
285 : WaitEventMaskForInstWithoutVscnt[T];
286 }
287
288 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
289};
290
291class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
292protected:
293 bool IsExpertMode;
294 static constexpr const HWEvents
295 WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
296 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
297 HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
298 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
299 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
300 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
301
302 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
303 HWEvents::VMEM_SAMPLER_READ_ACCESS,
304 HWEvents::VMEM_BVH_READ_ACCESS,
305
306 HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
307 HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
308 HWEvents::ASYNC_ACCESS,
309 HWEvents::TENSOR_ACCESS,
310 HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
311 HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
312 HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
313 HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
314 HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
315 HWEvents::VGPR_VMEM_READ};
316
317public:
318 WaitcntGeneratorGFX12Plus() = delete;
319 WaitcntGeneratorGFX12Plus(const MachineFunction &MF,
320 AMDGPU::InstCounterType MaxCounter,
321 const AMDGPU::HardwareLimits &Limits,
322 bool IsExpertMode)
323 : WaitcntGenerator(MF, MaxCounter, Limits), IsExpertMode(IsExpertMode) {}
324
325 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
326 MachineInstr &OldWaitcntInstr,
327 AMDGPU::Waitcnt &Wait,
329
330 bool createNewWaitcnt(MachineBasicBlock &Block,
332 AMDGPU::Waitcnt Wait,
333 const WaitcntBrackets &ScoreBrackets) override;
334
335 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
336 return WaitEventMaskForInstGFX12Plus[T];
337 }
338
339 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
340};
341
342// Flags indicating which counters should be flushed in a loop preheader.
343struct PreheaderFlushFlags {
344 bool FlushVmCnt = false;
345 bool FlushDsCnt = false;
346};
347
348class SIInsertWaitcnts {
349 DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
350 DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
351 MachineLoopInfo &MLI;
352 MachinePostDominatorTree &PDT;
353 AliasAnalysis *AA = nullptr;
354 MachineFunction &MF;
355
356 struct BlockInfo {
357 std::unique_ptr<WaitcntBrackets> Incoming;
358 bool Dirty = true;
359 BlockInfo() = default;
360 BlockInfo(BlockInfo &&) = default;
361 BlockInfo &operator=(BlockInfo &&) = default;
362 ~BlockInfo();
363 };
364
365 MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
366
367 bool ForceEmitWaitcnt[AMDGPU::NUM_INST_CNTS] = {};
368
369 std::unique_ptr<WaitcntGenerator> WCG;
370
371 // Remember call and return instructions in the function.
372 DenseSet<MachineInstr *> CallInsts;
373 DenseSet<MachineInstr *> ReturnInsts;
374
375 // Remember all S_ENDPGM instructions. The boolean flag is true if there might
376 // be outstanding stores but definitely no outstanding scratch stores, to help
377 // with insertion of DEALLOC_VGPRS messages.
378 DenseMap<MachineInstr *, bool> EndPgmInsts;
379
380 AMDGPU::HardwareLimits Limits;
381
382public:
383 const GCNSubtarget &ST;
384 const SIInstrInfo &TII;
385 const SIRegisterInfo &TRI;
386 const MachineRegisterInfo &MRI;
387 AMDGPU::InstCounterType SmemAccessCounter;
388 AMDGPU::InstCounterType MaxCounter;
389 bool IsExpertMode = false;
390 const bool TgSplit;
391
392 SIInsertWaitcnts(MachineLoopInfo &MLI, MachinePostDominatorTree &PDT,
394 : MLI(MLI), PDT(PDT), AA(AA), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()),
395 TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()),
396 MRI(MF.getRegInfo()),
397 TgSplit(ST.hasTgSplitSupport() &&
398 AMDGPU::isTgSplitEnabled(MF.getFunction())) {}
399
400 const AMDGPU::HardwareLimits &getLimits() const { return Limits; }
401
402 PreheaderFlushFlags getPreheaderFlushFlags(MachineLoop *ML,
403 const WaitcntBrackets &Brackets);
404 PreheaderFlushFlags isPreheaderToFlush(MachineBasicBlock &MBB,
405 const WaitcntBrackets &ScoreBrackets);
406 bool isVMEMOrFlatVMEM(const MachineInstr &MI) const;
407 bool isDSRead(const MachineInstr &MI) const;
408 bool mayStoreIncrementingDSCNT(const MachineInstr &MI) const;
409 bool run();
410
411 bool isVmemAccess(const MachineInstr &MI) const;
412 bool generateWaitcntInstBefore(MachineInstr &MI,
413 WaitcntBrackets &ScoreBrackets,
414 MachineInstr *OldWaitcntInstr,
415 PreheaderFlushFlags FlushFlags);
416 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
418 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
419 MachineInstr *OldWaitcntInstr);
420 void updateEventWaitcntAfter(MachineInstr &Inst,
421 WaitcntBrackets *ScoreBrackets);
422 bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
423 MachineBasicBlock *Block) const;
424 bool insertForcedWaitAfter(MachineInstr &Inst, MachineBasicBlock &Block,
425 WaitcntBrackets &ScoreBrackets);
426 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
427 WaitcntBrackets &ScoreBrackets);
428 /// Removes redundant Soft Xcnt Waitcnts in \p Block emitted by the Memory
429 /// Legalizer. Returns true if block was modified.
430 bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
431 void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
432 bool ExpertMode) const;
433 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const {
434 return WCG->getWaitEvents(T);
435 }
436 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
437 return WCG->getCounterFromEvent(E);
438 }
439};
440
441// This objects maintains the current score brackets of each wait counter, and
442// a per-register scoreboard for each wait counter.
443//
444// We also maintain the latest score for every event type that can change the
445// waitcnt in order to know if there are multiple types of events within
446// the brackets. When multiple types of event happen in the bracket,
447// wait count may get decreased out of order, therefore we need to put in
448// "s_waitcnt 0" before use.
449class WaitcntBrackets {
450public:
451 WaitcntBrackets(const SIInsertWaitcnts *Context) : Context(Context) {
452 assert(Context->TRI.getNumRegUnits() < REGUNITS_END);
453 }
454
455#ifndef NDEBUG
456 ~WaitcntBrackets() {
457 unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
458 for (auto &[ID, Val] : VMem) {
459 if (Val.empty())
460 ++NumUnusedVmem;
461 }
462 for (auto &[ID, Val] : SGPRs) {
463 if (Val.empty())
464 ++NumUnusedSGPRs;
465 }
466
467 if (NumUnusedVmem || NumUnusedSGPRs) {
468 errs() << "WaitcntBracket had unused entries at destruction time: "
469 << NumUnusedVmem << " VMem and " << NumUnusedSGPRs
470 << " SGPR unused entries\n";
471 std::abort();
472 }
473 }
474#endif
475
476 bool isSmemCounter(AMDGPU::InstCounterType T) const {
477 return T == Context->SmemAccessCounter || T == AMDGPU::X_CNT;
478 }
479
480 unsigned getOutstanding(AMDGPU::InstCounterType T) const {
481 return ScoreUBs[T] - ScoreLBs[T];
482 }
483
484 bool hasPendingVMEM(VMEMID ID, AMDGPU::InstCounterType T) const {
485 return getVMemScore(ID, T) > getScoreLB(T);
486 }
487
488 /// \Return true if we have no score entries for counter \p T.
489 bool empty(AMDGPU::InstCounterType T) const { return getScoreRange(T) == 0; }
490
491private:
492 unsigned getScoreLB(AMDGPU::InstCounterType T) const {
494 return ScoreLBs[T];
495 }
496
497 unsigned getScoreUB(AMDGPU::InstCounterType T) const {
499 return ScoreUBs[T];
500 }
501
502 unsigned getScoreRange(AMDGPU::InstCounterType T) const {
503 return getScoreUB(T) - getScoreLB(T);
504 }
505
506 unsigned getSGPRScore(MCRegUnit RU, AMDGPU::InstCounterType T) const {
507 auto It = SGPRs.find(RU);
508 return It != SGPRs.end() ? It->second.get(T) : 0;
509 }
510
511 unsigned getVMemScore(VMEMID TID, AMDGPU::InstCounterType T) const {
512 auto It = VMem.find(TID);
513 return It != VMem.end() ? It->second.Scores[T] : 0;
514 }
515
516 // Number of ordered XDL writes issued after the VA_VDST write tracked for
517 // TID. Only meaningful while that write is pending.
518 unsigned getNumOrderedXDLsAfterWrite(VMEMID TID) const {
519 return std::min(OrderedXDLCount - VMem.at(TID).OrderedXDLCountAtWrite,
520 getLimit(AMDGPU::VA_VDST_WR) - 1);
521 }
522
523public:
524 // Do some internal consistency checks.
525 void verify() {
526 // These track the same underlying hardware counter so the score ranges
527 // should be identical.
528 assert(getScoreLB(AMDGPU::VA_VDST_RD) == getScoreLB(AMDGPU::VA_VDST_WR));
529 assert(getScoreUB(AMDGPU::VA_VDST_RD) == getScoreUB(AMDGPU::VA_VDST_WR));
530 }
531
532 bool merge(const WaitcntBrackets &Other);
533
534 bool counterOutOfOrder(AMDGPU::InstCounterType T) const;
535 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
536 simplifyWaitcnt(Wait, Wait);
537 }
538 void simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
539 AMDGPU::Waitcnt &UpdateWait) const;
540 void simplifyWaitcnt(AMDGPU::InstCounterType T, unsigned &Count) const;
541 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) const;
542 void simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
543 AMDGPU::Waitcnt &UpdateWait) const;
544 void simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
545 AMDGPU::Waitcnt &UpdateWait) const;
546 // GLOBAL_INV/BUFFER_INV increment loadcnt/vmcnt but do not write VGPRs, so
547 // waits that only drain these events can be omitted.
548 void simplifyVmemInvOnlyWait(AMDGPU::Waitcnt &Wait) const {
549 // LOAD_CNT also includes store events on targets without VScnt.
550 HWEvents LoadEvents =
551 PendingEvents & Context->getWaitEvents(AMDGPU::LOAD_CNT);
552 if (LoadEvents == HWEvents::VMEM_INV_ACCESS)
553 Wait.clear(AMDGPU::LOAD_CNT);
554 }
555
556 void determineWaitForPhysReg(AMDGPU::InstCounterType T, MCRegister Reg,
557 AMDGPU::Waitcnt &Wait,
558 const MachineInstr &MI) const;
559 MCRegister determineVGPR16Dependency(const MachineInstr &MI,
561 MCRegister Reg) const;
562 void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
563 AMDGPU::Waitcnt &Wait) const;
564 AMDGPU::Waitcnt determineAsyncWait(unsigned N, AMDGPU::AsyncStages Stages);
565 void tryClearSCCWriteEvent(MachineInstr *Inst);
566
567 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
568 void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
569 void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
570 void updateByEvent(HWEvents E, MachineInstr &MI);
571 void recordAsyncMark(MachineInstr &MI, AMDGPU::AsyncStages Stages);
572
573 HWEvents getPendingEvents() const { return PendingEvents; }
574 bool hasPendingEvent() const { return PendingEvents.any(); }
575 bool hasPendingEvent(HWEvents E) const { return PendingEvents.contains(E); }
576 bool hasPendingEvent(AMDGPU::InstCounterType T) const {
577 bool HasPending = (PendingEvents & Context->getWaitEvents(T)).any();
578 assert(HasPending == !empty(T) &&
579 "Expected pending events iff scoreboard is not empty");
580 return HasPending;
581 }
582
583 bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
584 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
585 // Return true if more than one bit is set in Events.
586 return Events.size() > 1;
587 }
588
589 bool hasPendingFlat() const {
590 return ((LastFlatDsCnt > ScoreLBs[AMDGPU::DS_CNT] &&
591 LastFlatDsCnt <= ScoreUBs[AMDGPU::DS_CNT]) ||
592 (LastFlatLoadCnt > ScoreLBs[AMDGPU::LOAD_CNT] &&
593 LastFlatLoadCnt <= ScoreUBs[AMDGPU::LOAD_CNT]));
594 }
595
596 void setPendingFlat() {
597 LastFlatLoadCnt = ScoreUBs[AMDGPU::LOAD_CNT];
598 LastFlatDsCnt = ScoreUBs[AMDGPU::DS_CNT];
599 }
600
601 bool hasPendingXcntOnlyVmem() const {
602 return LastXcntOnlyVmem > ScoreLBs[AMDGPU::X_CNT] &&
603 LastXcntOnlyVmem <= ScoreUBs[AMDGPU::X_CNT];
604 }
605
606 void setPendingXcntOnlyVmem() { LastXcntOnlyVmem = ScoreUBs[AMDGPU::X_CNT]; }
607
608 // Return true if an xcnt event not tracked by asynccnt may still be
609 // outstanding after \p Wait. Waiting for a counter to reach zero completes
610 // its operations, which also drains their xcnt.
611 bool hasPendingNonAsyncXcnt(
612 const AMDGPU::Waitcnt &Wait = AMDGPU::Waitcnt()) const {
613 if (hasPendingXcntOnlyVmem() || (hasPendingEvent(HWEvents::SMEM_GROUP) &&
614 Wait.get(AMDGPU::KM_CNT) != 0))
615 return true;
618 if (Wait.get(T) != 0 && hasPendingEvent(T))
619 return true;
620 }
621 return false;
622 }
623
624 bool hasPendingGDS() const {
625 return LastGDS > ScoreLBs[AMDGPU::DS_CNT] &&
626 LastGDS <= ScoreUBs[AMDGPU::DS_CNT];
627 }
628
629 unsigned getPendingGDSWait() const {
630 return std::min(getScoreUB(AMDGPU::DS_CNT) - LastGDS,
631 getLimit(AMDGPU::DS_CNT) - 1);
632 }
633
634 void setPendingGDS() { LastGDS = ScoreUBs[AMDGPU::DS_CNT]; }
635
636 // Return true if there might be pending writes to the vgpr-interval by VMEM
637 // instructions where the HWEvents in VGPRContext are not contained in E.
638 bool hasDifferentVGPRPendingEvents(MCRegister Reg, HWEvents E) const {
639 for (MCRegUnit RU : regunits(Reg)) {
640 auto It = VMem.find(toVMEMID(RU));
641 if (It != VMem.end() && (It->second.VGPRPendingEvents & ~E).any())
642 return true;
643 }
644 return false;
645 }
646
647 void clearVGPRPendingEvents(MCRegister Reg) {
648 for (MCRegUnit RU : regunits(Reg)) {
649 if (auto It = VMem.find(toVMEMID(RU)); It != VMem.end()) {
650 It->second.VGPRPendingEvents = HWEvents::NONE;
651 if (It->second.empty())
652 VMem.erase(It);
653 }
654 }
655 }
656
657 void setStateOnFunctionEntryOrReturn() {
658 setScoreUB(AMDGPU::STORE_CNT,
659 getScoreUB(AMDGPU::STORE_CNT) + getLimit(AMDGPU::STORE_CNT));
660 PendingEvents |= Context->getWaitEvents(AMDGPU::STORE_CNT);
661 }
662
663 ArrayRef<const MachineInstr *> getLDSDMAStores() const {
664 return LDSDMAStores;
665 }
666
667 bool hasPointSampleAccel(const MachineInstr &MI) const;
668 bool hasPointSamplePendingVmemTypes(const MachineInstr &MI,
669 MCRegister RU) const;
670
671 void print(raw_ostream &) const;
672 void dump() const { print(dbgs()); }
673
674 // Free up memory by removing empty entries from the DenseMap that track event
675 // scores.
676 void purgeEmptyTrackingData();
677
678private:
679 unsigned getLimit(AMDGPU::InstCounterType T) const {
680 return Context->getLimits().get(T);
681 }
682
683 struct MergeInfo {
684 unsigned OldLB;
685 unsigned OtherLB;
686 unsigned MyShift;
687 unsigned OtherShift;
688 };
689
690 using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
691
692 unsigned getWaitForScore(AMDGPU::InstCounterType T, unsigned Score) const;
693 void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
694 AMDGPU::Waitcnt &Wait) const;
695 void determineWaitForVAVDSTWrite(VMEMID ID, AMDGPU::Waitcnt &Wait) const;
696
697 static bool mergeScore(const MergeInfo &M, unsigned &Score,
698 unsigned OtherScore);
699 bool mergeAsyncMarks(AMDGPU::AsyncStages Stage,
700 ArrayRef<MergeInfo> MergeInfos,
701 ArrayRef<CounterValueArray> OtherMarks);
702
703 iterator_range<MCRegUnitIterator> regunits(MCRegister Reg) const {
704 assert(Reg != AMDGPU::SCC && "Shouldn't be used on SCC");
705 if (!Context->TRI.isInAllocatableClass(Reg))
706 return {{}, {}};
707 return Context->TRI.regunits(Reg);
708 }
709
710 void setScoreLB(AMDGPU::InstCounterType T, unsigned Val) {
712 ScoreLBs[T] = Val;
713 }
714
715 void setScoreUB(AMDGPU::InstCounterType T, unsigned Val) {
717 ScoreUBs[T] = Val;
718
719 if (T != AMDGPU::EXP_CNT)
720 return;
721
722 if (getScoreRange(AMDGPU::EXP_CNT) > getLimit(AMDGPU::EXP_CNT))
723 ScoreLBs[AMDGPU::EXP_CNT] =
724 ScoreUBs[AMDGPU::EXP_CNT] - getLimit(AMDGPU::EXP_CNT);
725 }
726
727 void setRegScore(MCRegister Reg, AMDGPU::InstCounterType T, unsigned Val) {
728 const SIRegisterInfo &TRI = Context->TRI;
729 if (Reg == AMDGPU::SCC) {
730 SCCScore = Val;
731 } else if (TRI.isVectorRegister(Context->MRI, Reg)) {
732 for (MCRegUnit RU : regunits(Reg)) {
733 auto &Info = VMem[toVMEMID(RU)];
734 Info.Scores[T] = Val;
735 if (T == AMDGPU::VA_VDST_WR)
736 Info.OrderedXDLCountAtWrite = OrderedXDLCount;
737 }
738 } else if (TRI.isSGPRReg(Context->MRI, Reg)) {
739 for (MCRegUnit RU : regunits(Reg))
740 SGPRs[RU].get(T) = Val;
741 } else {
742 llvm_unreachable("Register cannot be tracked/unknown register!");
743 }
744 }
745
746 void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
747 VMem[TID].Scores[T] = Val;
748 }
749
750 void setScoreByOperand(const MachineOperand &Op,
751 AMDGPU::InstCounterType CntTy, unsigned Val);
752
753 const SIInsertWaitcnts *Context;
754
755 unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
756 unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
757 HWEvents PendingEvents;
758 // Remember the last flat memory operation.
759 unsigned LastFlatDsCnt = 0;
760 unsigned LastFlatLoadCnt = 0;
761 // Remember the last VMEM operation that increments xcnt but neither loadcnt
762 // nor storecnt, such as a prefetch or an LDS-only FLAT. Async LDS DMA is
763 // excluded: it is covered by ASYNC_ACCESS instead.
764 unsigned LastXcntOnlyVmem = 0;
765 // Remember the last GDS operation.
766 unsigned LastGDS = 0;
767 // Number of VA_VDST ordered XDL writes issued so far. Only differences
768 // between values are meaningful, so unsigned wraparound is harmless.
769 unsigned OrderedXDLCount = 0;
770
771 // The score tracking logic is fragmented as follows:
772 // - VMem: VGPR RegUnits and LDS DMA IDs, see the VMEMID encoding.
773 // - SGPRs: SGPR RegUnits
774 // - SCC: Non-allocatable and not general purpose: not a SGPR.
775 //
776 // For the VMem case, if the key is within the range of LDS DMA IDs,
777 // then the corresponding index into the `LDSDMAStores` vector below is:
778 // Key - LDSDMA_BEGIN - 1
779 // This is because LDSDMA_BEGIN is a generic entry and does not have an
780 // associated MachineInstr.
781 //
782 // TODO: Could we track SCC alongside SGPRs so it's not longer a special case?
783
784 struct VMEMInfo {
785 // Scores for all instruction counters. Zero-initialized.
786 CounterValueArray Scores{};
787 // OrderedXDLCount when the VA_VDST_WR score was set. Every ordered XDL
788 // write issued since then remains outstanding while this write is pending.
789 unsigned OrderedXDLCountAtWrite = 0;
790 // For VGPRs, we need to track an additional fine-grained set of pending
791 // events.
792 HWEvents VGPRPendingEvents;
793
794 bool empty() const {
795 return all_of(Scores, equal_to(0)) && !VGPRPendingEvents;
796 }
797 };
798
799 /// Wait cnt scores for every sgpr, the DS_CNT (corresponding to LGKMcnt
800 /// pre-gfx12) or KM_CNT (gfx12+ only), and X_CNT (gfx1250) are relevant.
801 class SGPRInfo {
802 /// Either DS_CNT or KM_CNT score.
803 unsigned ScoreDsKmCnt = 0;
804 unsigned ScoreXCnt = 0;
805
806 public:
807 unsigned get(AMDGPU::InstCounterType T) const {
808 assert(
809 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
810 "Invalid counter");
811 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
812 }
813 unsigned &get(AMDGPU::InstCounterType T) {
814 assert(
815 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
816 "Invalid counter");
817 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
818 }
819
820 bool empty() const { return !ScoreDsKmCnt && !ScoreXCnt; }
821 };
822
823 DenseMap<VMEMID, VMEMInfo> VMem; // VGPR + LDS DMA
824 DenseMap<MCRegUnit, SGPRInfo> SGPRs;
825
826 // Reg score for SCC.
827 unsigned SCCScore = 0;
828 // The unique instruction that has an SCC write pending, if there is one.
829 const MachineInstr *PendingSCCWrite = nullptr;
830
831 // Store representative LDS DMA operations. The only useful info here is
832 // alias info. One store is kept per unique AAInfo.
833 SmallVector<const MachineInstr *> LDSDMAStores;
834
835 // State of all counters at each async mark encountered so far, per stage.
836 std::array<SmallVector<CounterValueArray, 0>, AMDGPU::NumAsyncStages>
837 AsyncMarks;
838
839 // But in the rare pathological case, a nest of loops that pushes marks
840 // without waiting on any mark can cause AsyncMarks to grow very large. We cap
841 // it to a reasonable limit. We can tune this later or potentially introduce a
842 // user option to control the value.
843 static constexpr unsigned MaxAsyncMarks = 16;
844
845 // Track the upper bound score for async operations that are not part of a
846 // mark yet, per stage. Initialized to all zeros.
847 std::array<CounterValueArray, AMDGPU::NumAsyncStages> AsyncScore{};
848};
849
850SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
851
852class SIInsertWaitcntsLegacy : public MachineFunctionPass {
853public:
854 static char ID;
855 SIInsertWaitcntsLegacy() : MachineFunctionPass(ID) {}
856
857 bool runOnMachineFunction(MachineFunction &MF) override;
858
859 StringRef getPassName() const override {
860 return "SI insert wait instructions";
861 }
862
863 void getAnalysisUsage(AnalysisUsage &AU) const override {
864 AU.setPreservesCFG();
865 AU.addRequired<MachineLoopInfoWrapperPass>();
866 AU.addRequired<MachinePostDominatorTreeWrapperPass>();
867 AU.addUsedIfAvailable<AAResultsWrapperPass>();
868 AU.addPreserved<AAResultsWrapperPass>();
870 }
871};
872
873} // end anonymous namespace
874
875void WaitcntBrackets::setScoreByOperand(const MachineOperand &Op,
877 unsigned Score) {
878 setRegScore(Op.getReg().asMCReg(), CntTy, Score);
879}
880
881// Return true if the subtarget is one that enables Point Sample Acceleration
882// and the MachineInstr passed in is one to which it might be applied (the
883// hardware makes this decision based on several factors, but we can't determine
884// this at compile time, so we have to assume it might be applied if the
885// instruction supports it).
886bool WaitcntBrackets::hasPointSampleAccel(const MachineInstr &MI) const {
887 if (!Context->ST.hasPointSampleAccel() || !SIInstrInfo::isMIMG(MI))
888 return false;
889
890 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode());
891 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
893 return BaseInfo->PointSampleAccel;
894}
895
896// Return true if the subtarget enables Point Sample Acceleration, the supplied
897// MachineInstr is one to which it might be applied and the supplied interval is
898// one that has outstanding writes to vmem-types different than VMEM_NOSAMPLER
899// (this is the type that a point sample accelerated instruction effectively
900// becomes)
901bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
902 MCRegister Reg) const {
903 if (!hasPointSampleAccel(MI))
904 return false;
905
906 return hasDifferentVGPRPendingEvents(Reg, HWEvents::VMEM_READ_ACCESS);
907}
908
909void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
910 assert(E.size() == 1 && "Expected singular event!");
911 AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
912 assert(T < Context->MaxCounter);
913
914 unsigned UB = getScoreUB(T);
915 unsigned Increment = 1;
916 if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
918 Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
919 // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
920 // two VOP3P instructions and increments VA_VDST twice.
921 Increment = 2;
922 }
923 unsigned CurrScore = UB + Increment;
924 if (CurrScore <= UB)
925 report_fatal_error("InsertWaitcnt score wraparound");
926 // PendingEvents and ScoreUB need to be update regardless if this event
927 // changes the score of a register or not.
928 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
929 PendingEvents |= E;
930 setScoreUB(T, CurrScore);
931
932 // Count this write before recording its own destination, so the
933 // destination does not count itself. A VOP3PX2 WMMA counts once even though
934 // it increments VA_VDST twice: only its WMMA half is an ordered XDL write.
935 if (E == HWEvents::VGPR_XDL_WRITE && Context->TII.isVAVDSTOrderedXDL(Inst))
936 ++OrderedXDLCount;
937
938 const SIRegisterInfo &TRI = Context->TRI;
939 const MachineRegisterInfo &MRI = Context->MRI;
940 const SIInstrInfo &TII = Context->TII;
941
942 if (T == AMDGPU::EXP_CNT) {
943 // Put score on the source vgprs. If this is a store, just use those
944 // specific register(s).
945 if (TII.isDS(Inst) && Inst.mayLoadOrStore()) {
946 // All GDS operations must protect their address register (same as
947 // export.)
948 if (const auto *AddrOp = TII.getNamedOperand(Inst, AMDGPU::OpName::addr))
949 setScoreByOperand(*AddrOp, AMDGPU::EXP_CNT, CurrScore);
950
951 if (Inst.mayStore()) {
952 if (const auto *Data0 =
953 TII.getNamedOperand(Inst, AMDGPU::OpName::data0))
954 setScoreByOperand(*Data0, AMDGPU::EXP_CNT, CurrScore);
955 if (const auto *Data1 =
956 TII.getNamedOperand(Inst, AMDGPU::OpName::data1))
957 setScoreByOperand(*Data1, AMDGPU::EXP_CNT, CurrScore);
958 } else if (SIInstrInfo::isAtomicRet(Inst) && !SIInstrInfo::isGWS(Inst) &&
959 Inst.getOpcode() != AMDGPU::DS_APPEND &&
960 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
961 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
962 for (const MachineOperand &Op : Inst.all_uses()) {
963 if (TRI.isVectorRegister(MRI, Op.getReg()))
964 setScoreByOperand(Op, AMDGPU::EXP_CNT, CurrScore);
965 }
966 }
967 } else if (TII.isFLAT(Inst)) {
968 if (Inst.mayStore()) {
969 setScoreByOperand(*TII.getNamedOperand(Inst, AMDGPU::OpName::data),
970 AMDGPU::EXP_CNT, CurrScore);
971 } else if (SIInstrInfo::isAtomicRet(Inst)) {
972 setScoreByOperand(*TII.getNamedOperand(Inst, AMDGPU::OpName::data),
973 AMDGPU::EXP_CNT, CurrScore);
974 }
975 } else if (TII.isMIMG(Inst)) {
976 if (Inst.mayStore()) {
977 setScoreByOperand(Inst.getOperand(0), AMDGPU::EXP_CNT, CurrScore);
978 } else if (SIInstrInfo::isAtomicRet(Inst)) {
979 setScoreByOperand(*TII.getNamedOperand(Inst, AMDGPU::OpName::data),
980 AMDGPU::EXP_CNT, CurrScore);
981 }
982 } else if (TII.isMTBUF(Inst)) {
983 if (Inst.mayStore())
984 setScoreByOperand(Inst.getOperand(0), AMDGPU::EXP_CNT, CurrScore);
985 } else if (TII.isMUBUF(Inst)) {
986 if (Inst.mayStore()) {
987 setScoreByOperand(Inst.getOperand(0), AMDGPU::EXP_CNT, CurrScore);
988 } else if (SIInstrInfo::isAtomicRet(Inst)) {
989 setScoreByOperand(*TII.getNamedOperand(Inst, AMDGPU::OpName::data),
990 AMDGPU::EXP_CNT, CurrScore);
991 }
992 } else if (TII.isLDSDIR(Inst)) {
993 // LDSDIR instructions attach the score to the destination.
994 setScoreByOperand(*TII.getNamedOperand(Inst, AMDGPU::OpName::vdst),
995 AMDGPU::EXP_CNT, CurrScore);
996 } else {
997 if (TII.isEXP(Inst)) {
998 // For export the destination registers are really temps that
999 // can be used as the actual source after export patching, so
1000 // we need to treat them like sources and set the EXP_CNT
1001 // score.
1002 for (MachineOperand &DefMO : Inst.all_defs()) {
1003 if (TRI.isVGPR(MRI, DefMO.getReg())) {
1004 setScoreByOperand(DefMO, AMDGPU::EXP_CNT, CurrScore);
1005 }
1006 }
1007 }
1008 for (const MachineOperand &Op : Inst.all_uses()) {
1009 if (TRI.isVectorRegister(MRI, Op.getReg()))
1010 setScoreByOperand(Op, AMDGPU::EXP_CNT, CurrScore);
1011 }
1012 }
1013 } else if (T == AMDGPU::X_CNT) {
1014 HWEvents OtherEvent =
1015 E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
1016 if (PendingEvents.contains(OtherEvent)) {
1017 // Hardware inserts an implicit xcnt between interleaved
1018 // SMEM and VMEM operations. So there will never be
1019 // outstanding address translations for both SMEM and
1020 // VMEM at the same time.
1021 setScoreLB(T, getScoreUB(T) - 1);
1022 PendingEvents -= OtherEvent;
1023 }
1024 for (const MachineOperand &Op : Inst.all_uses()) {
1025 // Async ops have an implicit use of ASYNCcnt, which is not a real
1026 // register.
1027 if (Op.getReg() == AMDGPU::ASYNCcnt)
1028 continue;
1029 setScoreByOperand(Op, T, CurrScore);
1030 }
1031 } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
1032 T == AMDGPU::VM_VSRC) {
1033 // Match the score to the VGPR destination or source registers as
1034 // appropriate
1035 for (const MachineOperand &Op : Inst.operands()) {
1036 if (!Op.isReg())
1037 continue;
1038
1039 // Skip based on counter type and operand type
1040 if (T == AMDGPU::VA_VDST_RD && Op.isDef())
1041 continue; // RD tracks reads only
1042 if (T == AMDGPU::VA_VDST_WR && Op.isUse())
1043 continue; // WR tracks writes only
1044 if (T == AMDGPU::VM_VSRC && Op.isDef())
1045 continue;
1046
1047 if (TRI.isVectorRegister(Context->MRI, Op.getReg()))
1048 setScoreByOperand(Op, T, CurrScore);
1049 }
1050 } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
1051 // Match the score to the destination registers.
1052 //
1053 // Check only explicit operands. Stores, especially spill stores, include
1054 // implicit uses and defs of their super registers which would create an
1055 // artificial dependency, while these are there only for register liveness
1056 // accounting purposes.
1057 //
1058 // Special cases where implicit register defs exists, such as M0 or VCC,
1059 // but none with memory instructions.
1060 for (const MachineOperand &Op : Inst.defs()) {
1061 if (T == AMDGPU::LOAD_CNT || T == AMDGPU::SAMPLE_CNT ||
1062 T == AMDGPU::BVH_CNT) {
1063 if (!TRI.isVectorRegister(MRI, Op.getReg())) // TODO: add wrapper
1064 continue;
1065 if (updateVMCntOnly(Inst)) {
1066 // updateVMCntOnly should only leave us with VGPRs
1067 // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
1068 // defs.
1069 assert(TRI.isVectorRegister(MRI, Op.getReg()));
1070 HWEvents VGPRContext =
1072 // If instruction can have Point Sample Accel applied, we have to flag
1073 // this with another potential dependency
1074 if (hasPointSampleAccel(Inst))
1075 VGPRContext |= HWEvents::VMEM_READ_ACCESS;
1076 for (MCRegUnit RU : regunits(Op.getReg().asMCReg()))
1077 VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
1078 }
1079 }
1080 setScoreByOperand(Op, T, CurrScore);
1081 }
1082 if (Inst.mayStore() &&
1083 (TII.isDS(Inst) || TII.isNonAsyncLdsDmaWrite(Inst))) {
1084 // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
1085 // written can be accessed. A load from LDS to VMEM does not need a wait.
1086 //
1087 // The "Slot" is the offset from LDSDMA_BEGIN. If it's non-zero, then
1088 // there is a MachineInstr in LDSDMAStores used to track this LDSDMA
1089 // store. The "Slot" is the index into LDSDMAStores + 1.
1090 size_t Slot = 0;
1091 for (const auto *MemOp : Inst.memoperands()) {
1092 if (!MemOp->isStore() ||
1093 MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
1094 continue;
1095 // Comparing just AA info does not guarantee memoperands are equal
1096 // in general, but this is so for LDS DMA in practice.
1097 auto AAI = MemOp->getAAInfo();
1098 // Alias scope information gives a way to definitely identify an
1099 // original memory object and practically produced in the module LDS
1100 // lowering pass. If there is no scope available we will not be able
1101 // to disambiguate LDS aliasing as after the module lowering all LDS
1102 // is squashed into a single big object.
1103 if (!AAI || !AAI.Scope)
1104 break;
1105 for (size_t I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
1106 for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
1107 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
1108 Slot = I + 1;
1109 break;
1110 }
1111 }
1112 }
1113 if (Slot)
1114 break;
1115 // The slot may not be valid because it can be >= NUM_LDSDMA which
1116 // means the scoreboard cannot track it. We still want to preserve the
1117 // MI in order to check alias information, though.
1118 LDSDMAStores.push_back(&Inst);
1119 Slot = LDSDMAStores.size();
1120 break;
1121 }
1122 setVMemScore(LDSDMA_BEGIN, T, CurrScore);
1123 if (Slot && Slot < NUM_LDSDMA)
1124 setVMemScore(LDSDMA_BEGIN + static_cast<VMEMID>(Slot), T, CurrScore);
1125 }
1126
1127 AMDGPU::AsyncStages Stage = TII.getAsyncStage(Inst);
1128 if (any(Stage))
1129 AsyncScore[AMDGPU::getAsyncStageIndex(Stage)][T] = CurrScore;
1130
1132 setRegScore(AMDGPU::SCC, T, CurrScore);
1133 PendingSCCWrite = &Inst;
1134 }
1135 }
1136}
1137
1138void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst,
1139 AMDGPU::AsyncStages Stages) {
1140 // In the absence of loops, AsyncMarks can grow linearly with the program
1141 // until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
1142 // limit every time we push a new mark, but that seems like unnecessary work
1143 // in practical cases. We do separately truncate the array when processing a
1144 // loop, which should be sufficient.
1145 //
1146 // A mark is inserted in every stage named. Each stage gets a copy of its
1147 // corresponding score.
1148 LLVM_DEBUG(dbgs() << "recordAsyncMark(stages=" << Stages << "):\n" << Inst);
1149 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Stages)) {
1150 unsigned SI = AMDGPU::getAsyncStageIndex(S);
1151 AsyncMarks[SI].push_back(AsyncScore[SI]);
1152 LLVM_DEBUG({
1153 dbgs() << " stage " << S << ":\n";
1154 for (const auto &Mark : AsyncMarks[SI]) {
1155 dbgs() << " ";
1156 llvm::interleaveComma(Mark, dbgs());
1157 dbgs() << '\n';
1158 }
1159 });
1160 }
1161}
1162
1163void WaitcntBrackets::print(raw_ostream &OS) const {
1164 const GCNSubtarget &ST = Context->ST;
1165
1166 for (auto T : inst_counter_types(Context->MaxCounter)) {
1167 unsigned SR = getScoreRange(T);
1168 switch (T) {
1169 case AMDGPU::LOAD_CNT:
1170 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
1171 << SR << "):";
1172 break;
1173 case AMDGPU::DS_CNT:
1174 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
1175 << SR << "):";
1176 break;
1177 case AMDGPU::EXP_CNT:
1178 OS << " EXP_CNT(" << SR << "):";
1179 break;
1180 case AMDGPU::STORE_CNT:
1181 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
1182 << SR << "):";
1183 break;
1184 case AMDGPU::SAMPLE_CNT:
1185 OS << " SAMPLE_CNT(" << SR << "):";
1186 break;
1187 case AMDGPU::BVH_CNT:
1188 OS << " BVH_CNT(" << SR << "):";
1189 break;
1190 case AMDGPU::KM_CNT:
1191 OS << " KM_CNT(" << SR << "):";
1192 break;
1193 case AMDGPU::X_CNT:
1194 OS << " X_CNT(" << SR << "):";
1195 break;
1196 case AMDGPU::ASYNC_CNT:
1197 OS << " ASYNC_CNT(" << SR << "):";
1198 break;
1199 case AMDGPU::VA_VDST_RD:
1200 OS << " VA_VDST_RD(" << SR << "): ";
1201 break;
1202 case AMDGPU::VA_VDST_WR:
1203 OS << " VA_VDST_WR(" << SR << "): ";
1204 break;
1205 case AMDGPU::VM_VSRC:
1206 OS << " VM_VSRC(" << SR << "): ";
1207 break;
1208 default:
1209 OS << " UNKNOWN(" << SR << "):";
1210 break;
1211 }
1212
1213 if (SR != 0) {
1214 // Print vgpr scores.
1215 unsigned LB = getScoreLB(T);
1216
1217 SmallVector<VMEMID> SortedVMEMIDs(VMem.keys());
1218 sort(SortedVMEMIDs);
1219
1220 for (auto ID : SortedVMEMIDs) {
1221 unsigned RegScore = VMem.at(ID).Scores[T];
1222 if (RegScore <= LB)
1223 continue;
1224 unsigned RelScore = RegScore - LB - 1;
1225 if (ID < REGUNITS_END) {
1226 OS << ' ' << RelScore << ':'
1227 << printRegUnit(static_cast<MCRegUnit>(ID), &Context->TRI);
1228 } else {
1229 assert(ID >= LDSDMA_BEGIN && ID < LDSDMA_END &&
1230 "Unhandled/unexpected ID value!");
1231 OS << ' ' << RelScore << ":LDSDMA" << ID;
1232 }
1233 }
1234
1235 // Also need to print sgpr scores for lgkm_cnt or xcnt.
1236 if (isSmemCounter(T)) {
1237 SmallVector<MCRegUnit> SortedSMEMIDs(SGPRs.keys());
1238 sort(SortedSMEMIDs);
1239 for (auto ID : SortedSMEMIDs) {
1240 unsigned RegScore = SGPRs.at(ID).get(T);
1241 if (RegScore <= LB)
1242 continue;
1243 unsigned RelScore = RegScore - LB - 1;
1244 OS << ' ' << RelScore << ':'
1245 << printRegUnit(static_cast<MCRegUnit>(ID), &Context->TRI);
1246 }
1247 }
1248
1249 if (T == AMDGPU::KM_CNT && SCCScore > 0)
1250 OS << ' ' << SCCScore << ":scc";
1251 }
1252 OS << '\n';
1253 }
1254
1255 OS << "Pending Events: ";
1256 if (hasPendingEvent()) {
1257 OS << getPendingEvents();
1258 } else {
1259 OS << "none";
1260 }
1261 OS << '\n';
1262
1263 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(AMDGPU::AsyncStages::ALL)) {
1264 unsigned SI = AMDGPU::getAsyncStageIndex(S);
1265 const CounterValueArray &Score = AsyncScore[SI];
1266 OS << "Async score (stage " << S << "): ";
1267 if (llvm::all_of(Score, [](unsigned V) { return V == 0; }))
1268 OS << "none";
1269 else
1270 llvm::interleaveComma(Score, OS);
1271 OS << '\n';
1272
1273 OS << "Async marks (stage " << S << "): " << AsyncMarks[SI].size() << '\n';
1274
1275 for (const auto &Mark : AsyncMarks[SI]) {
1276 for (auto T : AMDGPU::inst_counter_types()) {
1277 unsigned MarkedScore = Mark[T];
1278 switch (T) {
1279 case AMDGPU::LOAD_CNT:
1280 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
1281 << "_CNT: " << MarkedScore;
1282 break;
1283 case AMDGPU::DS_CNT:
1284 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
1285 << "_CNT: " << MarkedScore;
1286 break;
1287 case AMDGPU::EXP_CNT:
1288 OS << " EXP_CNT: " << MarkedScore;
1289 break;
1290 case AMDGPU::STORE_CNT:
1291 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
1292 << "_CNT: " << MarkedScore;
1293 break;
1294 case AMDGPU::SAMPLE_CNT:
1295 OS << " SAMPLE_CNT: " << MarkedScore;
1296 break;
1297 case AMDGPU::BVH_CNT:
1298 OS << " BVH_CNT: " << MarkedScore;
1299 break;
1300 case AMDGPU::KM_CNT:
1301 OS << " KM_CNT: " << MarkedScore;
1302 break;
1303 case AMDGPU::X_CNT:
1304 OS << " X_CNT: " << MarkedScore;
1305 break;
1306 case AMDGPU::ASYNC_CNT:
1307 OS << " ASYNC_CNT: " << MarkedScore;
1308 break;
1309 default:
1310 OS << " UNKNOWN: " << MarkedScore;
1311 break;
1312 }
1313 }
1314 OS << '\n';
1315 }
1316 }
1317 OS << '\n';
1318}
1319
1320/// Simplify \p UpdateWait by removing waits that are redundant based on the
1321/// current WaitcntBrackets and any other waits specified in \p CheckWait.
1322void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
1323 AMDGPU::Waitcnt &UpdateWait) const {
1324 simplifyWaitcnt(UpdateWait, AMDGPU::LOAD_CNT);
1325 simplifyWaitcnt(UpdateWait, AMDGPU::EXP_CNT);
1326 simplifyWaitcnt(UpdateWait, AMDGPU::DS_CNT);
1327 simplifyWaitcnt(UpdateWait, AMDGPU::STORE_CNT);
1328 simplifyWaitcnt(UpdateWait, AMDGPU::SAMPLE_CNT);
1329 simplifyWaitcnt(UpdateWait, AMDGPU::BVH_CNT);
1330 simplifyWaitcnt(UpdateWait, AMDGPU::KM_CNT);
1331 simplifyXcnt(CheckWait, UpdateWait);
1332 simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST_RD);
1333 simplifyWaitcnt(UpdateWait, AMDGPU::VA_VDST_WR);
1334 simplifyVmVsrc(CheckWait, UpdateWait);
1335 simplifyWaitcnt(UpdateWait, AMDGPU::ASYNC_CNT);
1336}
1337
1338void WaitcntBrackets::simplifyWaitcnt(AMDGPU::InstCounterType T,
1339 unsigned &Count) const {
1340 // The number of outstanding events for this type, T, can be calculated
1341 // as (UB - LB). If the current Count is greater than or equal to the number
1342 // of outstanding events, then the wait for this counter is redundant.
1343 if (Count >= getScoreRange(T))
1344 Count = ~0u;
1345}
1346
1347void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait,
1348 AMDGPU::InstCounterType T) const {
1349 unsigned Cnt = Wait.get(T);
1350 simplifyWaitcnt(T, Cnt);
1351 Wait.set(T, Cnt);
1352}
1353
1354void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
1355 AMDGPU::Waitcnt &UpdateWait) const {
1356 // Try to simplify xcnt further by checking for joint kmcnt and loadcnt
1357 // optimizations. On entry to a block with multiple predescessors, there may
1358 // be pending SMEM and VMEM events active at the same time.
1359 // In such cases, only clear one active event at a time.
1360 // TODO: Revisit xcnt optimizations for gfx1250.
1361 // Wait on XCNT is redundant if we are already waiting for a load to complete.
1362 // SMEM can return out of order, so only omit XCNT wait if we are waiting till
1363 // zero.
1364 if (CheckWait.get(AMDGPU::KM_CNT) == 0 &&
1365 hasPendingEvent(HWEvents::SMEM_GROUP))
1366 UpdateWait.set(AMDGPU::X_CNT, ~0u);
1367 // If we have pending store we cannot optimize XCnt because we do not wait for
1368 // stores. VMEM loads retun in order, so if we only have loads XCnt is
1369 // decremented to the same number as LOADCnt. That does not hold while a
1370 // VMEM operation that LOADCnt does not count is in flight.
1371 if (CheckWait.get(AMDGPU::LOAD_CNT) != ~0u &&
1372 hasPendingEvent(HWEvents::VMEM_GROUP) &&
1373 !hasPendingEvent(AMDGPU::STORE_CNT) && !hasPendingXcntOnlyVmem() &&
1374 !hasPendingEvent(HWEvents::ASYNC_ACCESS) &&
1375 CheckWait.get(AMDGPU::X_CNT) >= CheckWait.get(AMDGPU::LOAD_CNT))
1376 UpdateWait.set(AMDGPU::X_CNT, ~0u);
1377 // asynccnt==0 completes every async op, which also drains their xcnt.
1378 if (CheckWait.get(AMDGPU::ASYNC_CNT) == 0 &&
1379 hasPendingEvent(HWEvents::ASYNC_ACCESS) &&
1380 !hasPendingNonAsyncXcnt(CheckWait))
1381 UpdateWait.set(AMDGPU::X_CNT, ~0u);
1382 simplifyWaitcnt(UpdateWait, AMDGPU::X_CNT);
1383}
1384
1385void WaitcntBrackets::simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
1386 AMDGPU::Waitcnt &UpdateWait) const {
1387 // Waiting for a VMEM counter (like LOAD_CNT) implies an equivalent wait for
1388 // VM_VSRC, because if the VMEM operation has completed then it must surely
1389 // have read its VGPR sources, but only if there are no other outstanding VMEM
1390 // operations that use a different counter (like SAMPLE_CNT).
1391 static constexpr AMDGPU::InstCounterType VmemCounters[] = {
1394 HWEvents VmemEvents = llvm::accumulate(
1395 VmemCounters, HWEvents(), [&](HWEvents Acc, AMDGPU::InstCounterType T) {
1396 return Acc | Context->getWaitEvents(T);
1397 });
1398 HWEvents PendingVmemEvents = PendingEvents & VmemEvents;
1399 for (AMDGPU::InstCounterType T : VmemCounters) {
1400 unsigned CheckCount = CheckWait.get(T);
1401 if (UpdateWait.get(AMDGPU::VM_VSRC) >= CheckCount &&
1402 (CheckCount == 0 || !counterOutOfOrder(T)) &&
1403 (PendingVmemEvents & ~Context->getWaitEvents(T)) == 0)
1404 UpdateWait.set(AMDGPU::VM_VSRC, ~0u);
1405 }
1406
1407 simplifyWaitcnt(UpdateWait, AMDGPU::VM_VSRC);
1408}
1409
1410void WaitcntBrackets::purgeEmptyTrackingData() {
1411 VMem.remove_if([](const auto &P) { return P.second.empty(); });
1412 SGPRs.remove_if([](const auto &P) { return P.second.empty(); });
1413}
1414
1415// Returns the wait needed for \p ScoreToWait, or ~0u if no wait is needed.
1416unsigned WaitcntBrackets::getWaitForScore(AMDGPU::InstCounterType T,
1417 unsigned ScoreToWait) const {
1418 const unsigned LB = getScoreLB(T);
1419 const unsigned UB = getScoreUB(T);
1420
1421 // If the score falls within the bracket, we need a waitcnt.
1422 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1423 if ((T == AMDGPU::LOAD_CNT || T == AMDGPU::DS_CNT) && hasPendingFlat() &&
1424 !Context->ST.hasFlatLgkmVMemCountInOrder()) {
1425 // If there is a pending FLAT operation, and this is a VMem or LGKM
1426 // waitcnt and the target can report early completion, then we need
1427 // to force a waitcnt 0.
1428 return 0;
1429 }
1430 if (counterOutOfOrder(T)) {
1431 // Counter can get decremented out-of-order when there
1432 // are multiple types event in the bracket. Also emit an s_wait counter
1433 // with a conservative value of 0 for the counter.
1434 return 0;
1435 }
1436 // If a counter has been maxed out avoid overflow by waiting for
1437 // MAX(CounterType) - 1 instead.
1438 return std::min(UB - ScoreToWait, getLimit(T) - 1);
1439 }
1440 return ~0u;
1441}
1442
1443void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
1444 unsigned ScoreToWait,
1445 AMDGPU::Waitcnt &Wait) const {
1446 Wait.add(T, getWaitForScore(T, ScoreToWait));
1447}
1448
1449void WaitcntBrackets::determineWaitForVAVDSTWrite(VMEMID ID,
1450 AMDGPU::Waitcnt &Wait) const {
1451 unsigned Count =
1452 getWaitForScore(AMDGPU::VA_VDST_WR, getVMemScore(ID, AMDGPU::VA_VDST_WR));
1453 if (Count == ~0u)
1454 return;
1455
1456 // The ordered-XDL count is a wait bound independent of exact event-class
1457 // ordering. Combine the bounds for this register unit before Wait.add()
1458 // takes the minimum across all required register-unit dependencies.
1460 std::max(Count, getNumOrderedXDLsAfterWrite(ID)));
1461}
1462
1463AMDGPU::Waitcnt
1464WaitcntBrackets::determineAsyncWait(unsigned N, AMDGPU::AsyncStages Stages) {
1465 // Trim each named stage to at most N marks. The dropped marks imply waits
1466 // that get accumulated into one Waitcnt,
1467 // so a wait covering several stages is the union of their requirements.
1468 AMDGPU::Waitcnt Wait;
1469 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Stages)) {
1470 auto &StageMarks = AsyncMarks[AMDGPU::getAsyncStageIndex(S)];
1471 LLVM_DEBUG({
1472 dbgs() << "Need " << N << " async marks in stage " << S << ". Found "
1473 << StageMarks.size() << ":\n";
1474 for (const auto &Mark : StageMarks) {
1475 llvm::interleaveComma(Mark, dbgs());
1476 dbgs() << '\n';
1477 }
1478 });
1479
1480 unsigned StageN = N;
1481 if (StageMarks.size() == MaxAsyncMarks) {
1482 // Enforcing MaxAsyncMarks here is unnecessary work because the size of
1483 // MaxAsyncMarks is linear when traversing straightline code. But we do
1484 // need to check if truncation may have occured at a merge, and adjust N
1485 // to ensure that a wait is generated.
1486 LLVM_DEBUG(
1487 dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
1488 StageN = std::min(StageN, (unsigned)MaxAsyncMarks - 1);
1489 }
1490
1491 if (StageMarks.size() <= StageN) {
1492 LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
1493 continue;
1494 }
1495
1496 size_t MarkIndex = StageMarks.size() - StageN - 1;
1497 const auto &RequiredMark = StageMarks[MarkIndex];
1499 determineWaitForScore(T, RequiredMark[T], Wait);
1500 }
1501
1502 // Immediately remove the waited mark and all older ones
1503 // This happens BEFORE the wait is actually inserted, which is fine
1504 // because we've already extracted the wait requirements
1505 LLVM_DEBUG({
1506 dbgs() << "Removing " << (MarkIndex + 1)
1507 << " async marks after determining wait\n";
1508 });
1509 StageMarks.erase(StageMarks.begin(), StageMarks.begin() + MarkIndex + 1);
1510 }
1511
1512 LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
1513 return Wait;
1514}
1515
1516// With D16Write32BitVgpr, D16 inst might be clobbered by events running on the
1517// other half 16bit.
1518//
1519// Replace VGPR16 to VGPR32 for wait check if:
1520// 1. MI is a VALU, and there is a wait event on the other half
1521// 2. MI is a LdSt, and there is a wait event on the other half from different
1522// order group
1523MCRegister WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
1525 MCRegister Reg) const {
1526 const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
1527 uint64_t Size = Context->TRI.getRegSizeInBits(*RC);
1528
1529 if (Size != 16 || !Context->ST.hasD16Writes32BitVgpr())
1530 return Reg;
1531
1532 // With D16Writes32BitVgpr, D16 Inst might clobber the whole vgpr32
1533 // check dependency on the other half
1534 Register Reg32 = Context->TRI.get32BitRegister(Reg);
1535 Register OtherHalf = Context->TRI.getSubReg(
1536 Reg32,
1537 AMDGPU::isHi16Reg(Reg, Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
1538
1539 AMDGPU::Waitcnt Wait;
1540 for (MCRegUnit RU : regunits(OtherHalf)) {
1541 if (T == AMDGPU::VA_VDST_WR)
1542 determineWaitForVAVDSTWrite(toVMEMID(RU), Wait);
1543 else
1544 determineWaitForScore(T, getVMemScore(toVMEMID(RU), T), Wait);
1545 }
1546
1547 // No wait on otherhalf
1548 if (!Wait.hasWait())
1549 return Reg;
1550
1551 if (Context->TII.isVALU(MI, /*AllowLDSDMA=*/false))
1552 return Reg32;
1553
1554 // If hi/lo16 mixed events
1555 HWEvents MIEvents = AMDGPU::getEventsFor(
1556 MI, Context->ST, Context->IsExpertMode, Context->TgSplit);
1557 HWEvents OtherHalfEvents = Context->getWaitEvents(T);
1558 HWEvents Events = MIEvents & OtherHalfEvents;
1559 if (Events.size() > 1)
1560 return Reg32;
1561 return Reg;
1562}
1563
1564void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
1565 MCRegister Reg,
1566 AMDGPU::Waitcnt &Wait,
1567 const MachineInstr &MI) const {
1568 if (Reg == AMDGPU::SCC) {
1569 determineWaitForScore(T, SCCScore, Wait);
1570 } else {
1571 bool IsVGPR = Context->TRI.isVectorRegister(Context->MRI, Reg);
1572 if (IsVGPR)
1573 Reg = determineVGPR16Dependency(MI, T, Reg);
1574 for (MCRegUnit RU : regunits(Reg)) {
1575 if (IsVGPR && T == AMDGPU::VA_VDST_WR) {
1576 determineWaitForVAVDSTWrite(toVMEMID(RU), Wait);
1577 } else {
1578 determineWaitForScore(
1579 T, IsVGPR ? getVMemScore(toVMEMID(RU), T) : getSGPRScore(RU, T),
1580 Wait);
1581 }
1582 }
1583 }
1584}
1585
1586void WaitcntBrackets::determineWaitForLDSDMA(AMDGPU::InstCounterType T,
1587 VMEMID TID,
1588 AMDGPU::Waitcnt &Wait) const {
1589 assert(TID >= LDSDMA_BEGIN && TID < LDSDMA_END);
1590 determineWaitForScore(T, getVMemScore(TID, T), Wait);
1591}
1592
1593void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
1594 // S_BARRIER_WAIT on the same barrier guarantees that the pending write to
1595 // SCC has landed
1596 if (PendingSCCWrite &&
1597 PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
1598 PendingSCCWrite->getOperand(0).getImm() == Inst->getOperand(0).getImm()) {
1599 HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
1600 // If this SCC_WRITE is the only pending KM_CNT event, clear counter.
1601 if ((PendingEvents & Context->getWaitEvents(AMDGPU::KM_CNT)) ==
1602 SCC_WRITE_PendingEvent) {
1603 setScoreLB(AMDGPU::KM_CNT, getScoreUB(AMDGPU::KM_CNT));
1604 }
1605
1606 PendingEvents -= SCC_WRITE_PendingEvent;
1607 PendingSCCWrite = nullptr;
1608 }
1609}
1610
1611void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
1613 unsigned Cnt;
1615 Cnt =
1616 std::min(Wait.get(AMDGPU::VA_VDST_RD), Wait.get(AMDGPU::VA_VDST_WR));
1617 } else {
1618 Cnt = Wait.get(T);
1619 }
1620 applyWaitcnt(T, Cnt);
1621 }
1622}
1623
1624void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
1625 const unsigned UB = getScoreUB(T);
1626 if (Count >= UB)
1627 return;
1628 if (Count != 0) {
1629 if (counterOutOfOrder(T))
1630 return;
1631 setScoreLB(T, std::max(getScoreLB(T), UB - Count));
1632 } else {
1633 setScoreLB(T, UB);
1634 PendingEvents -= Context->getWaitEvents(T);
1635 }
1636
1637 if (T == AMDGPU::KM_CNT && Count == 0 &&
1638 hasPendingEvent(HWEvents::SMEM_GROUP)) {
1639 if (!hasMixedPendingEvents(AMDGPU::X_CNT))
1640 applyWaitcnt(AMDGPU::X_CNT, 0);
1641 else
1642 PendingEvents -= HWEvents::SMEM_GROUP;
1643 }
1644 if (T == AMDGPU::LOAD_CNT && hasPendingEvent(HWEvents::VMEM_GROUP) &&
1645 !hasPendingEvent(AMDGPU::STORE_CNT) && !hasPendingXcntOnlyVmem() &&
1646 !hasPendingEvent(HWEvents::ASYNC_ACCESS)) {
1647 if (!hasMixedPendingEvents(AMDGPU::X_CNT))
1648 applyWaitcnt(AMDGPU::X_CNT, Count);
1649 else if (Count == 0)
1650 PendingEvents -= HWEvents::VMEM_GROUP;
1651 }
1652 if (T == AMDGPU::ASYNC_CNT && Count == 0 && !hasPendingNonAsyncXcnt())
1653 applyWaitcnt(AMDGPU::X_CNT, 0);
1654}
1655
1656void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
1658 unsigned Cnt = Wait.get(T);
1659 applyWaitcnt(T, Cnt);
1660}
1661
1662// Where there are multiple types of event in the bracket of a counter,
1663// the decrement may go out of order.
1664bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
1665 // Scalar memory read always can go out of order.
1666 if ((T == Context->SmemAccessCounter &&
1667 hasPendingEvent(HWEvents::SMEM_ACCESS)) ||
1668 (T == AMDGPU::X_CNT && hasPendingEvent(HWEvents::SMEM_GROUP)))
1669 return true;
1670
1671 if (T == AMDGPU::LOAD_CNT) {
1672
1673 // On targets without VScnt, LOAD_CNT includes all of STORE_CNT as well.
1674 // All these events use one counter and do not go out of order with respect
1675 // to each other.
1676 if (!Context->ST.hasVscnt())
1677 return false;
1678
1679 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
1680
1681 // If the target does not have extended counters, VMEM_BVH/SAMPLE_READ
1682 // events are equivalent to VMEM_READ_ACCESS. We do not go out of order in
1683 // such cases.
1684 static constexpr HWEvents ExtendedImageEvents =
1685 HWEvents::VMEM_SAMPLER_READ_ACCESS | HWEvents::VMEM_BVH_READ_ACCESS;
1686 if (!Context->ST.hasExtendedWaitCounts() &&
1687 (Events & ExtendedImageEvents).any()) {
1688 Events -= ExtendedImageEvents;
1689 Events |= HWEvents::VMEM_READ_ACCESS;
1690 }
1691
1692 // GLOBAL_INV/BUFFER_INV complete in-order with VMEM_READ_ACCESS, so mixing
1693 // VMEM_INV_ACCESS with other LOAD_CNT events does not cause out-of-order
1694 // completion.
1695 Events -= HWEvents::VMEM_INV_ACCESS;
1696
1697 return Events.size() > 1;
1698 }
1699
1700 return hasMixedPendingEvents(T);
1701}
1702
1703INITIALIZE_PASS_BEGIN(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1704 false, false)
1707INITIALIZE_PASS_END(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1709
1710char SIInsertWaitcntsLegacy::ID = 0;
1711
1712char &llvm::SIInsertWaitcntsID = SIInsertWaitcntsLegacy::ID;
1713
1715 return new SIInsertWaitcntsLegacy();
1716}
1717
1718static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName,
1719 unsigned NewEnc) {
1720 int OpIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
1721 assert(OpIdx >= 0);
1722
1723 MachineOperand &MO = MI.getOperand(OpIdx);
1724
1725 if (NewEnc == MO.getImm())
1726 return false;
1727
1728 MO.setImm(NewEnc);
1729 return true;
1730}
1731
1732bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt) const {
1733 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Waitcnt->getOpcode());
1734 if (Opcode == Waitcnt->getOpcode())
1735 return false;
1736
1737 Waitcnt->setDesc(TII.get(Opcode));
1738 return true;
1739}
1740
1741/// Combine consecutive S_WAITCNT and S_WAITCNT_VSCNT instructions that
1742/// precede \p It and follow \p OldWaitcntInstr and apply any extra waits
1743/// from \p Wait that were added by previous passes. Currently this pass
1744/// conservatively assumes that these preexisting waits are required for
1745/// correctness.
1746bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1747 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1748 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
1749 assert(isNormalMode(MaxCounter));
1750
1751 bool Modified = false;
1752 MachineInstr *WaitcntInstr = nullptr;
1753 MachineInstr *WaitcntVsCntInstr = nullptr;
1754
1755 LLVM_DEBUG({
1756 dbgs() << "PreGFX12::applyPreexistingWaitcnt at: ";
1757 if (It.isEnd())
1758 dbgs() << "end of block\n";
1759 else
1760 dbgs() << *It;
1761 });
1762
1763 for (auto &II :
1764 make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
1765 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1766 if (isNonWaitcntMetaInst(II)) {
1767 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1768 continue;
1769 }
1770
1771 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(II.getOpcode());
1772 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1773
1774 // Update required wait count. If this is a soft waitcnt (= it was added
1775 // by an earlier pass), it may be entirely removed.
1776 if (Opcode == AMDGPU::S_WAITCNT) {
1777 unsigned IEnc = static_cast<unsigned>(II.getOperand(0).getImm());
1778 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(IV, IEnc);
1779 if (TrySimplify) {
1780 ScoreBrackets.simplifyWaitcnt(OldWait);
1781 ScoreBrackets.simplifyVmemInvOnlyWait(OldWait);
1782 }
1783 Wait = Wait.combined(OldWait);
1784
1785 // Merge consecutive waitcnt of the same type by erasing multiples.
1786 if (WaitcntInstr) {
1787 DeferredErasures.erase(&II);
1788 II.eraseFromParent();
1789 Modified = true;
1790 } else if (!Wait.hasWaitExceptStoreCnt() && TrySimplify) {
1791 // Might or might not be redundant, depending on if there's a loop
1792 // carried dependency. Stash it away so we can erase it if nothing saves
1793 // it.
1794 DeferredErasures.insert(&II);
1795 } else {
1796 WaitcntInstr = &II;
1797 DeferredErasures.erase(&II);
1798 }
1799 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1800 assert(ST.hasVMemToLDSLoad());
1801 LLVM_DEBUG(dbgs() << "Processing S_WAITCNT_lds_direct: " << II
1802 << "Before: " << Wait << '\n';);
1803 ScoreBrackets.determineWaitForLDSDMA(AMDGPU::LOAD_CNT, LDSDMA_BEGIN,
1804 Wait);
1805 LLVM_DEBUG(dbgs() << "After: " << Wait << '\n';);
1806
1807 // It is possible (but unlikely) that this is the only wait instruction,
1808 // in which case, we exit this loop without a WaitcntInstr to consume
1809 // `Wait`. But that works because `Wait` was passed in by reference, and
1810 // the callee eventually calls createNewWaitcnt on it. We test this
1811 // possibility in an articial MIR test since such a situation cannot be
1812 // recreated by running the memory legalizer.
1813 II.eraseFromParent();
1814 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1815 unsigned N = static_cast<unsigned>(II.getOperand(0).getImm());
1816 LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
1817 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
1818 N, AMDGPU::asyncStagesFromMaskOperand(II.getOperand(1).getImm()));
1819 Wait = Wait.combined(OldWait);
1820 } else {
1821 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1822 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1823
1824 unsigned OldVSCnt = static_cast<unsigned>(
1825 TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm());
1826 if (TrySimplify)
1827 ScoreBrackets.simplifyWaitcnt(AMDGPU::STORE_CNT, OldVSCnt);
1829 std::min(Wait.get(AMDGPU::STORE_CNT), OldVSCnt));
1830
1831 if (WaitcntVsCntInstr) {
1832 DeferredErasures.erase(&II);
1833 II.eraseFromParent();
1834 Modified = true;
1835 } else if (!Wait.hasWaitStoreCnt() && TrySimplify) {
1836 // Loop-carried dependencies guard, see S_WAITCNT case above.
1837 DeferredErasures.insert(&II);
1838 } else {
1839 WaitcntVsCntInstr = &II;
1840 DeferredErasures.erase(&II);
1841 }
1842 }
1843 }
1844
1845 if (WaitcntInstr) {
1846 Modified |= updateOperandIfDifferent(*WaitcntInstr, AMDGPU::OpName::simm16,
1848 Modified |= promoteSoftWaitCnt(WaitcntInstr);
1849
1850 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::LOAD_CNT);
1851 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::EXP_CNT);
1852 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::DS_CNT);
1853 Wait.set(AMDGPU::LOAD_CNT, ~0u);
1854 Wait.set(AMDGPU::EXP_CNT, ~0u);
1855 Wait.set(AMDGPU::DS_CNT, ~0u);
1856
1857 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
1858 << "New Instr at block end: "
1859 << *WaitcntInstr << '\n'
1860 : dbgs() << "applied pre-existing waitcnt\n"
1861 << "Old Instr: " << *It
1862 << "New Instr: " << *WaitcntInstr << '\n');
1863 }
1864
1865 if (WaitcntVsCntInstr) {
1866 Modified |=
1867 updateOperandIfDifferent(*WaitcntVsCntInstr, AMDGPU::OpName::simm16,
1868 Wait.get(AMDGPU::STORE_CNT));
1869 Modified |= promoteSoftWaitCnt(WaitcntVsCntInstr);
1870
1871 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::STORE_CNT);
1872 Wait.set(AMDGPU::STORE_CNT, ~0u);
1873
1874 LLVM_DEBUG(It.isEnd()
1875 ? dbgs() << "applied pre-existing waitcnt\n"
1876 << "New Instr at block end: " << *WaitcntVsCntInstr
1877 << '\n'
1878 : dbgs() << "applied pre-existing waitcnt\n"
1879 << "Old Instr: " << *It
1880 << "New Instr: " << *WaitcntVsCntInstr << '\n');
1881 }
1882
1883 return Modified;
1884}
1885
1886/// Generate S_WAITCNT and/or S_WAITCNT_VSCNT instructions for any
1887/// required counters in \p Wait
1888bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1889 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
1890 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
1891 assert(isNormalMode(MaxCounter));
1892
1893 bool Modified = false;
1894 const DebugLoc &DL = Block.findDebugLoc(It);
1895
1896 // Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
1897 // single instruction while VScnt has its own instruction.
1898 if (Wait.hasWaitExceptStoreCnt()) {
1899 // If profiling expansion is enabled, emit an expanded sequence
1900 if (ExpandWaitcntProfiling) {
1901 // Check if any of the counters to be waited on are out-of-order.
1902 // If so, fall back to normal (non-expanded) behavior since expansion
1903 // would provide misleading profiling information.
1904 bool AnyOutOfOrder = false;
1905 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1906 unsigned WaitCnt = Wait.get(CT);
1907 if (WaitCnt != ~0u && ScoreBrackets.counterOutOfOrder(CT)) {
1908 AnyOutOfOrder = true;
1909 break;
1910 }
1911 }
1912
1913 if (AnyOutOfOrder) {
1914 // Fall back to non-expanded wait
1915 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait);
1916 BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT)).addImm(Enc);
1917 Modified = true;
1918 } else {
1919 // All counters are in-order, safe to expand
1920 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1921 unsigned WaitCnt = Wait.get(CT);
1922 if (WaitCnt == ~0u)
1923 continue;
1924
1925 unsigned Outstanding =
1926 std::min(ScoreBrackets.getOutstanding(CT), getLimit(CT) - 1);
1927 EmitExpandedWaitcnt(Outstanding, WaitCnt, [&](unsigned Count) {
1928 AMDGPU::Waitcnt W;
1929 W.set(CT, Count);
1930 BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT))
1932 });
1933 Modified = true;
1934 }
1935 }
1936 } else {
1937 // Normal behavior: emit single combined waitcnt
1938 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait);
1939 [[maybe_unused]] auto SWaitInst =
1940 BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT)).addImm(Enc);
1941 Modified = true;
1942
1943 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1944 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1945 dbgs() << "New Instr: " << *SWaitInst << '\n');
1946 }
1947 }
1948
1949 if (Wait.hasWaitStoreCnt()) {
1950 assert(ST.hasVscnt());
1951
1952 if (ExpandWaitcntProfiling && Wait.get(AMDGPU::STORE_CNT) != ~0u &&
1953 !ScoreBrackets.counterOutOfOrder(AMDGPU::STORE_CNT)) {
1954 // Only expand if counter is not out-of-order
1955 unsigned Outstanding =
1956 std::min(ScoreBrackets.getOutstanding(AMDGPU::STORE_CNT),
1957 getLimit(AMDGPU::STORE_CNT) - 1);
1958 EmitExpandedWaitcnt(
1959 Outstanding, Wait.get(AMDGPU::STORE_CNT), [&](unsigned Count) {
1960 BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT_VSCNT))
1961 .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1962 .addImm(Count);
1963 });
1964 Modified = true;
1965 } else {
1966 [[maybe_unused]] auto SWaitInst =
1967 BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT_VSCNT))
1968 .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1970 Modified = true;
1971
1972 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1973 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1974 dbgs() << "New Instr: " << *SWaitInst << '\n');
1975 }
1976 }
1977
1978 return Modified;
1979}
1980
1981AMDGPU::Waitcnt
1982WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1983 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST.hasVscnt() ? 0 : ~0u);
1984}
1985
1986AMDGPU::Waitcnt
1987WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1988 unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
1989 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
1990 ~0u /* XCNT */, ~0u /* ASYNC_CNT */,
1991 ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal, ExpertVal);
1992}
1993
1994/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
1995/// follow \p OldWaitcntInstr and apply any extra waits from \p Wait that
1996/// were added by previous passes. Currently this pass conservatively
1997/// assumes that these preexisting waits are required for correctness.
1998bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1999 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
2000 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
2001 assert(!isNormalMode(MaxCounter));
2002
2003 bool Modified = false;
2004 MachineInstr *CombinedLoadDsCntInstr = nullptr;
2005 MachineInstr *CombinedStoreDsCntInstr = nullptr;
2006 MachineInstr *WaitcntDepctrInstr = nullptr;
2007 MachineInstr *WaitInstrs[AMDGPU::NUM_EXTENDED_INST_CNTS] = {};
2008
2009 LLVM_DEBUG({
2010 dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: ";
2011 if (It.isEnd())
2012 dbgs() << "end of block\n";
2013 else
2014 dbgs() << *It;
2015 });
2016
2017 // Accumulate waits that should not be simplified.
2018 AMDGPU::Waitcnt RequiredWait;
2019
2020 for (auto &II :
2021 make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
2022 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
2023 if (isNonWaitcntMetaInst(II)) {
2024 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
2025 continue;
2026 }
2027
2028 // Update required wait count. If this is a soft waitcnt (= it was added
2029 // by an earlier pass), it may be entirely removed.
2030
2031 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(II.getOpcode());
2032 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
2033
2034 // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
2035 // attempt to do more than that either.
2036 if (Opcode == AMDGPU::S_WAITCNT)
2037 continue;
2038
2039 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
2040 unsigned OldEnc = static_cast<unsigned>(
2041 TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm());
2042 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(IV, OldEnc);
2043 if (TrySimplify)
2044 Wait = Wait.combined(OldWait);
2045 else
2046 RequiredWait = RequiredWait.combined(OldWait);
2047 // Keep the first wait_loadcnt, erase the rest.
2048 if (CombinedLoadDsCntInstr == nullptr) {
2049 CombinedLoadDsCntInstr = &II;
2050 } else {
2051 II.eraseFromParent();
2052 Modified = true;
2053 }
2054 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
2055 unsigned OldEnc = static_cast<unsigned>(
2056 TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm());
2057 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(IV, OldEnc);
2058 if (TrySimplify)
2059 Wait = Wait.combined(OldWait);
2060 else
2061 RequiredWait = RequiredWait.combined(OldWait);
2062 // Keep the first wait_storecnt, erase the rest.
2063 if (CombinedStoreDsCntInstr == nullptr) {
2064 CombinedStoreDsCntInstr = &II;
2065 } else {
2066 II.eraseFromParent();
2067 Modified = true;
2068 }
2069 } else if (Opcode == AMDGPU::S_WAITCNT_DEPCTR) {
2070 unsigned OldEnc = static_cast<unsigned>(
2071 TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm());
2072 AMDGPU::Waitcnt OldWait;
2073 // Set both counters to the decoded value from the single hardware field
2074 unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(OldEnc);
2075 OldWait.set(AMDGPU::VA_VDST_RD, VaVdst);
2076 OldWait.set(AMDGPU::VA_VDST_WR, VaVdst);
2078 if (TrySimplify)
2079 ScoreBrackets.simplifyWaitcnt(OldWait);
2080 Wait = Wait.combined(OldWait);
2081 if (WaitcntDepctrInstr == nullptr) {
2082 WaitcntDepctrInstr = &II;
2083 } else {
2084 // S_WAITCNT_DEPCTR requires special care. Don't remove a
2085 // duplicate if it is waiting on things other than VA_VDST or
2086 // VM_VSRC. If that is the case, just make sure the VA_VDST and
2087 // VM_VSRC subfields of the operand are set to the "no wait"
2088 // values.
2089
2090 unsigned Enc = static_cast<unsigned>(
2091 TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm());
2092 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Enc, ~0u);
2093 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2094 unsigned VaVdst = std::min(Wait.get(AMDGPU::VA_VDST_RD),
2096 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
2097
2098 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(ST)) {
2099 Modified |= updateOperandIfDifferent(II, AMDGPU::OpName::simm16, Enc);
2100 Modified |= promoteSoftWaitCnt(&II);
2101 } else {
2102 II.eraseFromParent();
2103 Modified = true;
2104 }
2105 }
2106 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
2107 // Architectures higher than GFX10 do not have direct loads to
2108 // LDS, so no work required here yet.
2109 II.eraseFromParent();
2110 Modified = true;
2111 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
2112 // Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
2113 // shows up in the assembly as a comment with the original parameter N.
2114 unsigned N = static_cast<unsigned>(II.getOperand(0).getImm());
2115 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
2116 N, AMDGPU::asyncStagesFromMaskOperand(II.getOperand(1).getImm()));
2117 Wait = Wait.combined(OldWait);
2118 } else {
2119 std::optional<AMDGPU::InstCounterType> CT =
2121 assert(CT.has_value());
2122 unsigned OldCnt = static_cast<unsigned>(
2123 TII.getNamedOperand(II, AMDGPU::OpName::simm16)->getImm());
2124 if (TrySimplify)
2125 Wait.add(CT.value(), OldCnt);
2126 else
2127 RequiredWait.add(CT.value(), OldCnt);
2128 // Keep the first wait of its kind, erase the rest.
2129 if (WaitInstrs[CT.value()] == nullptr) {
2130 WaitInstrs[CT.value()] = &II;
2131 } else {
2132 II.eraseFromParent();
2133 Modified = true;
2134 }
2135 }
2136 }
2137
2138 ScoreBrackets.simplifyWaitcnt(Wait.combined(RequiredWait), Wait);
2139 Wait = Wait.combined(RequiredWait);
2140
2141 if (CombinedLoadDsCntInstr) {
2142 // Only keep an S_WAIT_LOADCNT_DSCNT if both counters actually need
2143 // to be waited for. Otherwise, let the instruction be deleted so
2144 // the appropriate single counter wait instruction can be inserted
2145 // instead, when new S_WAIT_*CNT instructions are inserted by
2146 // createNewWaitcnt(). As a side effect, resetting the wait counts will
2147 // cause any redundant S_WAIT_LOADCNT or S_WAIT_DSCNT to be removed by
2148 // the loop below that deals with single counter instructions.
2149 //
2150 // A wait for LOAD_CNT or DS_CNT implies a wait for VM_VSRC, since
2151 // instructions that have decremented LOAD_CNT or DS_CNT on completion
2152 // will have needed to wait for their register sources to be available
2153 // first.
2154 if (Wait.get(AMDGPU::LOAD_CNT) != ~0u && Wait.get(AMDGPU::DS_CNT) != ~0u) {
2155 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(IV, Wait);
2156 Modified |= updateOperandIfDifferent(*CombinedLoadDsCntInstr,
2157 AMDGPU::OpName::simm16, NewEnc);
2158 Modified |= promoteSoftWaitCnt(CombinedLoadDsCntInstr);
2159 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::LOAD_CNT);
2160 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::DS_CNT);
2161 Wait.set(AMDGPU::LOAD_CNT, ~0u);
2162 Wait.set(AMDGPU::DS_CNT, ~0u);
2163
2164 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2165 << "New Instr at block end: "
2166 << *CombinedLoadDsCntInstr << '\n'
2167 : dbgs() << "applied pre-existing waitcnt\n"
2168 << "Old Instr: " << *It << "New Instr: "
2169 << *CombinedLoadDsCntInstr << '\n');
2170 } else {
2171 CombinedLoadDsCntInstr->eraseFromParent();
2172 Modified = true;
2173 }
2174 }
2175
2176 if (CombinedStoreDsCntInstr) {
2177 // Similarly for S_WAIT_STORECNT_DSCNT.
2178 if (Wait.get(AMDGPU::STORE_CNT) != ~0u && Wait.get(AMDGPU::DS_CNT) != ~0u) {
2179 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(IV, Wait);
2180 Modified |= updateOperandIfDifferent(*CombinedStoreDsCntInstr,
2181 AMDGPU::OpName::simm16, NewEnc);
2182 Modified |= promoteSoftWaitCnt(CombinedStoreDsCntInstr);
2183 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::STORE_CNT);
2184 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::DS_CNT);
2185 Wait.set(AMDGPU::STORE_CNT, ~0u);
2186 Wait.set(AMDGPU::DS_CNT, ~0u);
2187
2188 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2189 << "New Instr at block end: "
2190 << *CombinedStoreDsCntInstr << '\n'
2191 : dbgs() << "applied pre-existing waitcnt\n"
2192 << "Old Instr: " << *It << "New Instr: "
2193 << *CombinedStoreDsCntInstr << '\n');
2194 } else {
2195 CombinedStoreDsCntInstr->eraseFromParent();
2196 Modified = true;
2197 }
2198 }
2199
2200 // Look for an opportunity to convert existing S_WAIT_LOADCNT,
2201 // S_WAIT_STORECNT and S_WAIT_DSCNT into new S_WAIT_LOADCNT_DSCNT
2202 // or S_WAIT_STORECNT_DSCNT. This is achieved by selectively removing
2203 // instructions so that createNewWaitcnt() will create new combined
2204 // instructions to replace them.
2205
2206 if (Wait.get(AMDGPU::DS_CNT) != ~0u) {
2207 // This is a vector of addresses in WaitInstrs pointing to instructions
2208 // that should be removed if they are present.
2210
2211 // If it's known that both DScnt and either LOADcnt or STOREcnt (but not
2212 // both) need to be waited for, ensure that there are no existing
2213 // individual wait count instructions for these.
2214
2215 if (Wait.get(AMDGPU::LOAD_CNT) != ~0u) {
2216 WaitsToErase.push_back(&WaitInstrs[AMDGPU::LOAD_CNT]);
2217 WaitsToErase.push_back(&WaitInstrs[AMDGPU::DS_CNT]);
2218 } else if (Wait.get(AMDGPU::STORE_CNT) != ~0u) {
2219 WaitsToErase.push_back(&WaitInstrs[AMDGPU::STORE_CNT]);
2220 WaitsToErase.push_back(&WaitInstrs[AMDGPU::DS_CNT]);
2221 }
2222
2223 for (MachineInstr **WI : WaitsToErase) {
2224 if (!*WI)
2225 continue;
2226
2227 DeferredErasures.erase(*WI);
2228 (*WI)->eraseFromParent();
2229 *WI = nullptr;
2230 Modified = true;
2231 }
2232 }
2233
2235 if (!WaitInstrs[CT])
2236 continue;
2237
2238 unsigned NewCnt = Wait.get(CT);
2239 if (NewCnt != ~0u) {
2240 Modified |= updateOperandIfDifferent(*WaitInstrs[CT],
2241 AMDGPU::OpName::simm16, NewCnt);
2242 Modified |= promoteSoftWaitCnt(WaitInstrs[CT]);
2243 DeferredErasures.erase(WaitInstrs[CT]);
2244
2245 ScoreBrackets.applyWaitcnt(CT, NewCnt);
2246 Wait.clear(CT);
2247
2248 LLVM_DEBUG(It.isEnd()
2249 ? dbgs() << "applied pre-existing waitcnt\n"
2250 << "New Instr at block end: " << *WaitInstrs[CT]
2251 << '\n'
2252 : dbgs() << "applied pre-existing waitcnt\n"
2253 << "Old Instr: " << *It
2254 << "New Instr: " << *WaitInstrs[CT] << '\n');
2255 } else {
2256 // Defer erasure in case there's a loop-carried dependency.
2257 DeferredErasures.insert(WaitInstrs[CT]);
2258 }
2259 }
2260
2261 if (WaitcntDepctrInstr) {
2262 // Get the encoded Depctr immediate and override the VA_VDST and VM_VSRC
2263 // subfields with the new required values.
2264 unsigned Enc = static_cast<unsigned>(
2265 TII.getNamedOperand(*WaitcntDepctrInstr, AMDGPU::OpName::simm16)
2266 ->getImm());
2268 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2269 unsigned VaVdst =
2270 std::min(Wait.get(AMDGPU::VA_VDST_RD), Wait.get(AMDGPU::VA_VDST_WR));
2271 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
2272
2273 ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST_RD, VaVdst);
2274 ScoreBrackets.applyWaitcnt(AMDGPU::VA_VDST_WR, VaVdst);
2275 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::VM_VSRC);
2276 Wait.set(AMDGPU::VA_VDST_RD, ~0u);
2277 Wait.set(AMDGPU::VA_VDST_WR, ~0u);
2278 Wait.set(AMDGPU::VM_VSRC, ~0u);
2279
2280 // If that new encoded Depctr immediate would actually still wait
2281 // for anything, update the instruction's operand. Otherwise it can
2282 // just be deleted.
2283 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(ST)) {
2284 Modified |= updateOperandIfDifferent(*WaitcntDepctrInstr,
2285 AMDGPU::OpName::simm16, Enc);
2286 LLVM_DEBUG(It.isEnd() ? dbgs() << "applyPreexistingWaitcnt\n"
2287 << "New Instr at block end: "
2288 << *WaitcntDepctrInstr << '\n'
2289 : dbgs() << "applyPreexistingWaitcnt\n"
2290 << "Old Instr: " << *It << "New Instr: "
2291 << *WaitcntDepctrInstr << '\n');
2292 } else {
2293 WaitcntDepctrInstr->eraseFromParent();
2294 Modified = true;
2295 }
2296 }
2297
2298 return Modified;
2299}
2300
2301/// Generate S_WAIT_*CNT instructions for any required counters in \p Wait
2302bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
2303 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
2304 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
2305 assert(!isNormalMode(MaxCounter));
2306
2307 bool Modified = false;
2308 const DebugLoc &DL = Block.findDebugLoc(It);
2309
2310 // For GFX12+, we use separate wait instructions, which makes expansion
2311 // simpler
2312 if (ExpandWaitcntProfiling) {
2314 unsigned Count = Wait.get(CT);
2315 if (Count == ~0u)
2316 continue;
2317
2318 // Skip expansion for out-of-order counters - emit normal wait instead
2319 if (ScoreBrackets.counterOutOfOrder(CT)) {
2320 BuildMI(Block, It, DL, TII.get(instrsForExtendedCounterTypes[CT]))
2321 .addImm(Count);
2322 Modified = true;
2323 continue;
2324 }
2325
2326 unsigned Outstanding =
2327 std::min(ScoreBrackets.getOutstanding(CT), getLimit(CT) - 1);
2328 EmitExpandedWaitcnt(Outstanding, Count, [&](unsigned Val) {
2329 BuildMI(Block, It, DL, TII.get(instrsForExtendedCounterTypes[CT]))
2330 .addImm(Val);
2331 });
2332 Modified = true;
2333 }
2334 return Modified;
2335 }
2336
2337 // Normal behavior (no expansion)
2338 // Check for opportunities to use combined wait instructions.
2339 if (Wait.get(AMDGPU::DS_CNT) != ~0u) {
2340 MachineInstr *SWaitInst = nullptr;
2341
2342 if (Wait.get(AMDGPU::LOAD_CNT) != ~0u) {
2343 unsigned Enc = AMDGPU::encodeLoadcntDscnt(IV, Wait);
2344
2345 SWaitInst = BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAIT_LOADCNT_DSCNT))
2346 .addImm(Enc);
2347
2348 Wait.set(AMDGPU::LOAD_CNT, ~0u);
2349 Wait.set(AMDGPU::DS_CNT, ~0u);
2350 } else if (Wait.get(AMDGPU::STORE_CNT) != ~0u) {
2351 unsigned Enc = AMDGPU::encodeStorecntDscnt(IV, Wait);
2352
2353 SWaitInst = BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAIT_STORECNT_DSCNT))
2354 .addImm(Enc);
2355
2356 Wait.set(AMDGPU::STORE_CNT, ~0u);
2357 Wait.set(AMDGPU::DS_CNT, ~0u);
2358 }
2359
2360 if (SWaitInst) {
2361 Modified = true;
2362
2363 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2364 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2365 dbgs() << "New Instr: " << *SWaitInst << '\n');
2366 }
2367 }
2368
2369 // Generate an instruction for any remaining counter that needs
2370 // waiting for.
2371
2373 unsigned Count = Wait.get(CT);
2374 if (Count == ~0u)
2375 continue;
2376
2377 [[maybe_unused]] auto SWaitInst =
2378 BuildMI(Block, It, DL, TII.get(instrsForExtendedCounterTypes[CT]))
2379 .addImm(Count);
2380
2381 Modified = true;
2382
2383 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2384 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2385 dbgs() << "New Instr: " << *SWaitInst << '\n');
2386 }
2387
2388 if (Wait.hasWaitDepctr()) {
2389 assert(IsExpertMode);
2390 unsigned Enc =
2392 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2393 unsigned VaVdst =
2394 std::min(Wait.get(AMDGPU::VA_VDST_RD), Wait.get(AMDGPU::VA_VDST_WR));
2395 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Enc, VaVdst);
2396
2397 [[maybe_unused]] auto SWaitInst =
2398 BuildMI(Block, It, DL, TII.get(AMDGPU::S_WAITCNT_DEPCTR)).addImm(Enc);
2399
2400 Modified = true;
2401
2402 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
2403 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2404 dbgs() << "New Instr: " << *SWaitInst << '\n');
2405 }
2406
2407 return Modified;
2408}
2409
2410/// Generate s_waitcnt instruction to be placed before cur_Inst.
2411/// Instructions of a given type are returned in order,
2412/// but instructions of different types can complete out of order.
2413/// We rely on this in-order completion
2414/// and simply assign a score to the memory access instructions.
2415/// We keep track of the active "score bracket" to determine
2416/// if an access of a memory read requires an s_waitcnt
2417/// and if so what the value of each counter is.
2418/// The "score bracket" is bound by the lower bound and upper bound
2419/// scores (*_score_LB and *_score_ub respectively).
2420/// If FlushFlags.FlushVmCnt is true, we want to flush the vmcnt counter here.
2421/// If FlushFlags.FlushDsCnt is true, we want to flush the dscnt counter here
2422/// (GFX12+ only, where DS_CNT is a separate counter).
2423bool SIInsertWaitcnts::generateWaitcntInstBefore(
2424 MachineInstr &MI, WaitcntBrackets &ScoreBrackets,
2425 MachineInstr *OldWaitcntInstr, PreheaderFlushFlags FlushFlags) {
2426 LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
2427
2428 assert(!isNonWaitcntMetaInst(MI));
2429
2430 AMDGPU::Waitcnt Wait;
2431 const unsigned Opc = MI.getOpcode();
2432
2433 switch (Opc) {
2434 case AMDGPU::BUFFER_WBINVL1:
2435 case AMDGPU::BUFFER_WBINVL1_SC:
2436 case AMDGPU::BUFFER_WBINVL1_VOL:
2437 case AMDGPU::BUFFER_GL0_INV:
2438 case AMDGPU::BUFFER_GL1_INV: {
2439 // FIXME: This should have already been handled by the memory legalizer.
2440 // Removing this currently doesn't affect any lit tests, but we need to
2441 // verify that nothing was relying on this. The number of buffer invalidates
2442 // being handled here should not be expanded.
2443 Wait.set(AMDGPU::LOAD_CNT, 0);
2444 break;
2445 }
2446 case AMDGPU::SI_RETURN_TO_EPILOG:
2447 case AMDGPU::SI_RETURN:
2448 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
2449 case AMDGPU::S_SETPC_B64_return: {
2450 // All waits must be resolved at call return.
2451 // NOTE: this could be improved with knowledge of all call sites or
2452 // with knowledge of the called routines.
2453 ReturnInsts.insert(&MI);
2454 AMDGPU::Waitcnt AllZeroWait =
2455 WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2456 ScoreBrackets.simplifyVmemInvOnlyWait(AllZeroWait);
2457 Wait = AllZeroWait;
2458 break;
2459 }
2460 case AMDGPU::S_ENDPGM:
2461 case AMDGPU::S_ENDPGM_SAVED: {
2462 // In dynamic VGPR mode, we want to release the VGPRs before the wave exits.
2463 // Technically the hardware will do this on its own if we don't, but that
2464 // might cost extra cycles compared to doing it explicitly.
2465 // When not in dynamic VGPR mode, identify S_ENDPGM instructions which may
2466 // have to wait for outstanding VMEM stores. In this case it can be useful
2467 // to send a message to explicitly release all VGPRs before the stores have
2468 // completed, but it is only safe to do this if there are no outstanding
2469 // scratch stores.
2470 EndPgmInsts[&MI] =
2471 !ScoreBrackets.empty(AMDGPU::STORE_CNT) &&
2472 !ScoreBrackets.hasPendingEvent(HWEvents::SCRATCH_WRITE_ACCESS);
2473 break;
2474 }
2475 case AMDGPU::S_SENDMSG:
2476 case AMDGPU::S_SENDMSGHALT: {
2477 if (ST.hasLegacyGeometry() &&
2478 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) ==
2480 // Resolve vm waits before gs-done.
2481 Wait.set(AMDGPU::LOAD_CNT, 0);
2482 break;
2483 }
2484 [[fallthrough]];
2485 }
2486 default: {
2487
2488 // Export & GDS instructions do not read the EXEC mask until after the
2489 // export is granted (which can occur well after the instruction is issued).
2490 // The shader program must flush all EXP operations on the export-count
2491 // before overwriting the EXEC mask.
2492 if (MI.modifiesRegister(AMDGPU::EXEC, &TRI)) {
2493 // Export and GDS are tracked individually, either may trigger a waitcnt
2494 // for EXEC.
2495 if (ScoreBrackets.hasPendingEvent(HWEvents::EXP_GPR_LOCK) ||
2496 ScoreBrackets.hasPendingEvent(HWEvents::EXP_PARAM_ACCESS) ||
2497 ScoreBrackets.hasPendingEvent(HWEvents::EXP_POS_ACCESS) ||
2498 ScoreBrackets.hasPendingEvent(HWEvents::GDS_GPR_LOCK)) {
2499 Wait.set(AMDGPU::EXP_CNT, 0);
2500 }
2501 }
2502
2503 // Wait for any pending GDS instruction to complete before any
2504 // "Always GDS" instruction.
2505 if (TII.isAlwaysGDS(Opc) && ScoreBrackets.hasPendingGDS())
2506 Wait.add(AMDGPU::DS_CNT, ScoreBrackets.getPendingGDSWait());
2507
2508 if (MI.isCall()) {
2509 // The function is going to insert a wait on everything in its prolog.
2510 // This still needs to be careful if the call target is a load (e.g. a GOT
2511 // load). We also need to check WAW dependency with saved PC.
2512 CallInsts.insert(&MI);
2513 Wait = AMDGPU::Waitcnt();
2514
2515 const MachineOperand &CallAddrOp = TII.getCalleeOperand(MI);
2516 if (CallAddrOp.isReg()) {
2517 ScoreBrackets.determineWaitForPhysReg(
2518 SmemAccessCounter, CallAddrOp.getReg().asMCReg(), Wait, MI);
2519
2520 if (const auto *RtnAddrOp =
2521 TII.getNamedOperand(MI, AMDGPU::OpName::dst)) {
2522 ScoreBrackets.determineWaitForPhysReg(
2523 SmemAccessCounter, RtnAddrOp->getReg().asMCReg(), Wait, MI);
2524 }
2525 }
2526 } else if (Opc == AMDGPU::S_BARRIER_WAIT) {
2527 ScoreBrackets.tryClearSCCWriteEvent(&MI);
2528 } else {
2529 // FIXME: Should not be relying on memoperands.
2530 // Look at the source operands of every instruction to see if
2531 // any of them results from a previous memory operation that affects
2532 // its current usage. If so, an s_waitcnt instruction needs to be
2533 // emitted.
2534 // If the source operand was defined by a load, add the s_waitcnt
2535 // instruction.
2536 //
2537 // Two cases are handled for destination operands:
2538 // 1) If the destination operand was defined by a load, add the s_waitcnt
2539 // instruction to guarantee the right WAW order.
2540 // 2) If a destination operand that was used by a recent export/store ins,
2541 // add s_waitcnt on exp_cnt to guarantee the WAR order.
2542
2543 for (const MachineMemOperand *Memop : MI.memoperands()) {
2544 const Value *Ptr = Memop->getValue();
2545 if (Memop->isStore()) {
2546 if (auto It = SLoadAddresses.find(Ptr); It != SLoadAddresses.end()) {
2547 Wait.add(SmemAccessCounter, 0);
2548 if (PDT.dominates(MI.getParent(), It->second))
2549 SLoadAddresses.erase(It);
2550 }
2551 }
2552 unsigned AS = Memop->getAddrSpace();
2554 continue;
2555 // No need to wait before load from VMEM to LDS.
2556 if (TII.mayWriteLDSThroughDMA(MI))
2557 continue;
2558
2559 // LOAD_CNT is only relevant to vgpr or LDS.
2560 VMEMID TID = LDSDMA_BEGIN;
2561 if (Ptr && Memop->getAAInfo()) {
2562 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
2563 for (size_t I = 0, E = LDSDMAStores.size(); I != E; ++I) {
2564 if (MI.mayAlias(AA, *LDSDMAStores[I], true)) {
2565 size_t Slot = I + 1;
2566 if (Slot >= NUM_LDSDMA) {
2567 // We didn't have enough slot to track this LDS DMA store, it
2568 // has been tracked using the common RegNo (FIRST_LDS_VGPR).
2569 ScoreBrackets.determineWaitForLDSDMA(AMDGPU::LOAD_CNT, TID,
2570 Wait);
2571 break;
2572 }
2573
2574 ScoreBrackets.determineWaitForLDSDMA(
2575 AMDGPU::LOAD_CNT, TID + static_cast<VMEMID>(Slot), Wait);
2576 }
2577 }
2578 } else {
2579 ScoreBrackets.determineWaitForLDSDMA(AMDGPU::LOAD_CNT, TID, Wait);
2580 }
2581 if (Memop->isStore()) {
2582 ScoreBrackets.determineWaitForLDSDMA(AMDGPU::EXP_CNT, TID, Wait);
2583 }
2584 }
2585
2586 // Loop over use and def operands.
2587 for (const MachineOperand &Op : MI.operands()) {
2588 if (!Op.isReg())
2589 continue;
2590
2591 // If the instruction does not read tied source, skip the operand.
2592 if (Op.isTied() && Op.isUse() && TII.doesNotReadTiedSource(MI))
2593 continue;
2594
2595 MCRegister Reg = Op.getReg().asMCReg();
2596
2597 const bool IsVGPR = TRI.isVectorRegister(MRI, Op.getReg());
2598 if (IsVGPR) {
2599 // Implicit VGPR defs and uses are never a part of the memory
2600 // instructions description and usually present to account for
2601 // super-register liveness.
2602 // TODO: Most of the other instructions also have implicit uses
2603 // for the liveness accounting only.
2604 if (Op.isImplicit() && MI.mayLoadOrStore())
2605 continue;
2606
2607 ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST_WR, Reg, Wait,
2608 MI);
2609 if (Op.isDef()) {
2610 ScoreBrackets.determineWaitForPhysReg(AMDGPU::VA_VDST_RD, Reg, Wait,
2611 MI);
2612 ScoreBrackets.determineWaitForPhysReg(AMDGPU::VM_VSRC, Reg, Wait,
2613 MI);
2614 }
2615
2616 // RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
2617 // previous write and this write are the same type of VMEM
2618 // instruction, in which case they are (in some architectures)
2619 // guaranteed to write their results in order anyway.
2620 // Additionally check instructions where Point Sample Acceleration
2621 // might be applied.
2622 if (Op.isUse() || !updateVMCntOnly(MI) ||
2623 ScoreBrackets.hasDifferentVGPRPendingEvents(
2625 ScoreBrackets.hasPointSamplePendingVmemTypes(MI, Reg) ||
2626 !ST.hasVmemWriteVgprInOrder()) {
2627 ScoreBrackets.determineWaitForPhysReg(AMDGPU::LOAD_CNT, Reg, Wait,
2628 MI);
2629 ScoreBrackets.determineWaitForPhysReg(AMDGPU::SAMPLE_CNT, Reg, Wait,
2630 MI);
2631 ScoreBrackets.determineWaitForPhysReg(AMDGPU::BVH_CNT, Reg, Wait,
2632 MI);
2633 ScoreBrackets.clearVGPRPendingEvents(Reg);
2634 }
2635
2636 if (Op.isDef() ||
2637 ScoreBrackets.hasPendingEvent(HWEvents::EXP_LDS_ACCESS)) {
2638 ScoreBrackets.determineWaitForPhysReg(AMDGPU::EXP_CNT, Reg, Wait,
2639 MI);
2640 }
2641 ScoreBrackets.determineWaitForPhysReg(AMDGPU::DS_CNT, Reg, Wait, MI);
2642 } else if (Op.getReg() == AMDGPU::SCC) {
2643 ScoreBrackets.determineWaitForPhysReg(AMDGPU::KM_CNT, Reg, Wait, MI);
2644 } else {
2645 ScoreBrackets.determineWaitForPhysReg(SmemAccessCounter, Reg, Wait,
2646 MI);
2647 }
2648
2649 if (ST.hasWaitXcnt() && ST.isXNACKEnabled() && Op.isDef())
2650 ScoreBrackets.determineWaitForPhysReg(AMDGPU::X_CNT, Reg, Wait, MI);
2651 }
2652 }
2653 }
2654 }
2655
2656 // Ensure safety against exceptions from outstanding memory operations while
2657 // waiting for a barrier:
2658 //
2659 // * Some subtargets safely handle backing off the barrier in hardware
2660 // when an exception occurs.
2661 // * Some subtargets have an implicit S_WAITCNT 0 before barriers, so that
2662 // there can be no outstanding memory operations during the wait.
2663 // * Subtargets with split barriers don't need to back off the barrier; it
2664 // is up to the trap handler to preserve the user barrier state correctly.
2665 //
2666 // In all other cases, ensure safety by ensuring that there are no outstanding
2667 // memory operations.
2668 if (Opc == AMDGPU::S_BARRIER && !ST.hasAutoWaitcntBeforeBarrier() &&
2669 !ST.hasBackOffBarrier()) {
2670 Wait = Wait.combined(WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
2671 }
2672
2673 // TODO: Remove this work-around, enable the assert for Bug 457939
2674 // after fixing the scheduler. Also, the Shader Compiler code is
2675 // independent of target.
2676 if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
2677 ScoreBrackets.hasPendingEvent(HWEvents::SMEM_ACCESS)) {
2678 Wait.set(AMDGPU::DS_CNT, 0);
2679 }
2680
2681 // Verify that the wait is actually needed.
2682 ScoreBrackets.simplifyWaitcnt(Wait);
2683
2684 // It is only necessary to insert an S_WAITCNT_DEPCTR instruction that
2685 // waits on VA_VDST if the instruction it would precede is not a VALU
2686 // instruction, since hardware handles VALU->VGPR->VALU hazards in
2687 // expert scheduling mode.
2688 if (TII.isVALU(MI, /*AllowLDSDMA=*/false)) {
2689 Wait.set(AMDGPU::VA_VDST_RD, ~0u);
2690 Wait.set(AMDGPU::VA_VDST_WR, ~0u);
2691 }
2692
2693 // Since the translation for VMEM addresses occur in-order, we can apply the
2694 // XCnt if the current instruction is of VMEM type and has a memory
2695 // dependency with another VMEM instruction in flight.
2696 if (Wait.get(AMDGPU::X_CNT) != ~0u && isVmemAccess(MI)) {
2697 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::X_CNT);
2698 Wait.set(AMDGPU::X_CNT, ~0u);
2699 }
2700
2701 // When forcing emit, we need to skip terminators because that would break the
2702 // terminators of the MBB if we emit a waitcnt between terminators.
2703 if (ST.getCLOpts().waitcnt_forcezero && !MI.isTerminator())
2704 Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2705
2706 // If we force waitcnt then update Wait accordingly.
2708 if (!ForceEmitWaitcnt[T])
2709 continue;
2710 Wait.set(T, 0);
2711 }
2712
2713 if (FlushFlags.FlushVmCnt) {
2716 Wait.set(T, 0);
2717 }
2718
2719 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(AMDGPU::DS_CNT))
2720 Wait.set(AMDGPU::DS_CNT, 0);
2721
2722 if (ST.getCLOpts().waitcnt_load_forcezero &&
2723 Wait.get(AMDGPU::LOAD_CNT) != ~0u)
2724 Wait.set(AMDGPU::LOAD_CNT, 0);
2725
2726 return generateWaitcnt(Wait, MI.getIterator(), *MI.getParent(), ScoreBrackets,
2727 OldWaitcntInstr);
2728}
2729
2730bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
2732 MachineBasicBlock &Block,
2733 WaitcntBrackets &ScoreBrackets,
2734 MachineInstr *OldWaitcntInstr) {
2735 bool Modified = false;
2736
2737 if (OldWaitcntInstr)
2738 // Try to merge the required wait with preexisting waitcnt instructions.
2739 // Also erase redundant waitcnt.
2740 Modified =
2741 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr, Wait, It);
2742
2743 // ExpCnt can be merged into VINTERP.
2744 if (Wait.get(AMDGPU::EXP_CNT) != ~0u && It != Block.instr_end() &&
2746 MachineOperand *WaitExp = TII.getNamedOperand(*It, AMDGPU::OpName::waitexp);
2747 if (Wait.get(AMDGPU::EXP_CNT) < WaitExp->getImm()) {
2748 WaitExp->setImm(Wait.get(AMDGPU::EXP_CNT));
2749 Modified = true;
2750 }
2751 // Apply ExpCnt before resetting it, so applyWaitcnt below sees all counts.
2752 ScoreBrackets.applyWaitcnt(Wait, AMDGPU::EXP_CNT);
2753 Wait.set(AMDGPU::EXP_CNT, ~0u);
2754
2755 LLVM_DEBUG(dbgs() << "generateWaitcnt\n"
2756 << "Update Instr: " << *It);
2757 }
2758
2759 if (WCG->createNewWaitcnt(Block, It, Wait, ScoreBrackets))
2760 Modified = true;
2761
2762 // Any counts that could have been applied to any existing waitcnt
2763 // instructions will have been done so, now deal with any remaining.
2764 ScoreBrackets.applyWaitcnt(Wait);
2765
2766 return Modified;
2767}
2768
2769bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
2770 return (TII.isFLAT(MI) && TII.mayAccessVMEMThroughFlat(MI)) ||
2771 (TII.isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(MI.getOpcode()));
2772}
2773
2774// Return true if the next instruction is S_ENDPGM, following fallthrough
2775// blocks if necessary.
2776bool SIInsertWaitcnts::isNextENDPGM(MachineBasicBlock::instr_iterator It,
2777 MachineBasicBlock *Block) const {
2778 auto BlockEnd = Block->getParent()->end();
2779 auto BlockIter = Block->getIterator();
2780
2781 while (true) {
2782 if (It.isEnd()) {
2783 if (++BlockIter != BlockEnd) {
2784 It = BlockIter->instr_begin();
2785 continue;
2786 }
2787
2788 return false;
2789 }
2790
2791 if (!It->isMetaInstruction())
2792 break;
2793
2794 It++;
2795 }
2796
2797 assert(!It.isEnd());
2798
2799 return It->getOpcode() == AMDGPU::S_ENDPGM;
2800}
2801
2802// Add a wait after an instruction if architecture requirements mandate one.
2803bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
2804 MachineBasicBlock &Block,
2805 WaitcntBrackets &ScoreBrackets) {
2806 AMDGPU::Waitcnt Wait;
2807 bool NeedsEndPGMCheck = false;
2808
2809 if (ST.isPreciseMemoryEnabled() && Inst.mayLoadOrStore())
2810 Wait = WCG->getAllZeroWaitcnt(Inst.mayStore() &&
2812
2813 if (TII.isAlwaysGDS(Inst.getOpcode())) {
2814 Wait.set(AMDGPU::DS_CNT, 0);
2815 NeedsEndPGMCheck = true;
2816 }
2817
2818 ScoreBrackets.simplifyWaitcnt(Wait);
2819
2820 auto SuccessorIt = next_nodbg(Inst.getIterator(), Block.instr_end());
2821 bool Result = generateWaitcnt(Wait, SuccessorIt, Block, ScoreBrackets,
2822 /*OldWaitcntInstr=*/nullptr);
2823
2824 if (Result && NeedsEndPGMCheck && isNextENDPGM(SuccessorIt, &Block)) {
2825 BuildMI(Block, SuccessorIt, Inst.getDebugLoc(), TII.get(AMDGPU::S_NOP))
2826 .addImm(0);
2827 }
2828
2829 return Result;
2830}
2831
2832void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
2833 WaitcntBrackets *ScoreBrackets) {
2834
2835 HWEvents InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode, TgSplit);
2836 for (HWEvents E : InstEvents)
2837 ScoreBrackets->updateByEvent(E, Inst);
2838
2839 if (TII.isDS(Inst) && TII.usesLGKM_CNT(Inst)) {
2840 if (TII.isAlwaysGDS(Inst.getOpcode()) ||
2841 TII.hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
2842 ScoreBrackets->setPendingGDS();
2843 }
2844 } else if (TII.isFLAT(Inst)) {
2845 if (InstEvents.contains(HWEvents::VMEM_GROUP) &&
2846 !TII.mayAccessVMEMThroughFlat(Inst) &&
2847 !InstEvents.contains(HWEvents::ASYNC_ACCESS))
2848 ScoreBrackets->setPendingXcntOnlyVmem();
2849
2850 if (Inst.mayLoadOrStore() && TII.mayAccessVMEMThroughFlat(Inst) &&
2851 TII.mayAccessLDSThroughFlat(Inst, TgSplit) &&
2852 !SIInstrInfo::isLDSDMA(Inst)) {
2853 // Async/LDSDMA operations have FLAT encoding but do not actually use flat
2854 // pointers. They do have two operands that each access global and LDS,
2855 // thus making it appear at this point that they are using a flat pointer.
2856 // Filter them out, and for the rest, generate a dependency on flat
2857 // pointers so that both VM and LGKM counters are flushed.
2858 ScoreBrackets->setPendingFlat();
2859 }
2860 } else if (Inst.isCall()) {
2861 // Act as a wait on everything, but AsyncCnt and TensorCnt are never
2862 // included in such blanket waits.
2863 ScoreBrackets->applyWaitcnt(WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
2864 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2865 } else if (TII.isVINTERP(Inst)) {
2866 int64_t Imm = TII.getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
2867 ScoreBrackets->applyWaitcnt(AMDGPU::EXP_CNT, static_cast<unsigned>(Imm));
2868 }
2869
2870 // Set XCNT to zero in the bracket for instructions that implicitly drain
2871 // XCNT.
2872 if (ST.hasWaitXcnt() && ST.isXNACKEnabled() && SIInstrInfo::isXcntDrain(Inst))
2873 ScoreBrackets->applyWaitcnt(AMDGPU::X_CNT, 0);
2874}
2875
2876bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
2877 unsigned OtherScore) {
2878 unsigned MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
2879 unsigned OtherShifted =
2880 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
2881 Score = std::max(MyShifted, OtherShifted);
2882 return OtherShifted > MyShifted;
2883}
2884
2885bool WaitcntBrackets::mergeAsyncMarks(AMDGPU::AsyncStages Stage,
2886 ArrayRef<MergeInfo> MergeInfos,
2887 ArrayRef<CounterValueArray> OtherMarks) {
2888 bool StrictDom = false;
2889 auto &StageMarks = AsyncMarks[AMDGPU::getAsyncStageIndex(Stage)];
2890
2891 LLVM_DEBUG(dbgs() << "Merging async marks of stage " << Stage << " ...");
2892 // Early exit: nothing to merge when both sides are empty.
2893 if (StageMarks.empty() && OtherMarks.empty()) {
2894 LLVM_DEBUG(dbgs() << " nothing to merge\n");
2895 return false;
2896 }
2897 LLVM_DEBUG(dbgs() << '\n');
2898
2899 // Determine maximum length needed after merging
2900 auto MaxSize = (unsigned)std::max(StageMarks.size(), OtherMarks.size());
2901 MaxSize = std::min(MaxSize, MaxAsyncMarks);
2902
2903 // Keep only the most recent marks within our limit.
2904 if (StageMarks.size() > MaxSize)
2905 StageMarks.erase(StageMarks.begin(),
2906 StageMarks.begin() + (StageMarks.size() - MaxSize));
2907
2908 // Pad with zero-filled marks if our list is shorter. Zero represents "no
2909 // pending async operations at this checkpoint" and acts as the identity
2910 // element for max() during merging. We pad at the beginning since the marks
2911 // need to be aligned in most-recent order.
2912 constexpr CounterValueArray ZeroMark{};
2913 StageMarks.insert(StageMarks.begin(), MaxSize - StageMarks.size(), ZeroMark);
2914
2915 LLVM_DEBUG({
2916 dbgs() << "Before merge:\n";
2917 for (const auto &Mark : StageMarks) {
2918 llvm::interleaveComma(Mark, dbgs());
2919 dbgs() << '\n';
2920 }
2921 dbgs() << "Other marks:\n";
2922 for (const auto &Mark : OtherMarks) {
2923 llvm::interleaveComma(Mark, dbgs());
2924 dbgs() << '\n';
2925 }
2926 });
2927
2928 // Merge element-wise using the existing mergeScore function and the
2929 // appropriate MergeInfo for each counter type. Rebase and merge EVERY
2930 // surviving mark into the new (widened) frame, aligned by recency from the
2931 // back. A mark with no counterpart on the other side merges against the
2932 // zero/identity mark: mergeScore still applies MyShift, so the mark is
2933 // re-expressed in the new frame instead of being left with a stale
2934 // (pre-merge) score.
2935 const size_t OtherSize = OtherMarks.size();
2936 const size_t OurSize = StageMarks.size();
2937 // After the both-empty early-return above, max(StageMarks, OtherMarks) >= 1,
2938 // and the erase/pad steps normalize StageMarks to exactly MaxSize. Hence
2939 // OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
2940 // seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
2941 // the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
2942 assert(OurSize >= 1 &&
2943 "StageMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
2944
2945 for (auto Idx : seq_inclusive<size_t>(1, OurSize)) {
2946 const CounterValueArray &OtherMark =
2947 Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
2948 for (auto T : inst_counter_types(Context->MaxCounter))
2949 StrictDom |=
2950 mergeScore(MergeInfos[T], StageMarks[OurSize - Idx][T], OtherMark[T]);
2951 }
2952
2953 LLVM_DEBUG({
2954 dbgs() << "After merge:\n";
2955 for (const auto &Mark : StageMarks) {
2956 llvm::interleaveComma(Mark, dbgs());
2957 dbgs() << '\n';
2958 }
2959 });
2960
2961 return StrictDom;
2962}
2963
2964/// Merge the pending events and associater score brackets of \p Other into
2965/// this brackets status.
2966///
2967/// Returns whether the merge resulted in a change that requires tighter waits
2968/// (i.e. the merged brackets strictly dominate the original brackets).
2969bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
2970 bool StrictDom = false;
2971
2972 // Check if "other" has keys we don't have, and create default entries for
2973 // those. If they remain empty after merging, we will clean it up after.
2974 for (auto K : Other.VMem.keys())
2975 VMem.try_emplace(K);
2976 for (auto K : Other.SGPRs.keys())
2977 SGPRs.try_emplace(K);
2978
2979 // Array to store MergeInfo for each counter type
2980 MergeInfo MergeInfos[AMDGPU::NUM_INST_CNTS];
2981
2982 for (auto T : inst_counter_types(Context->MaxCounter)) {
2983 // Merge event flags for this counter
2984 const HWEvents &EventsForT = Context->getWaitEvents(T);
2985 const HWEvents OldEvents = PendingEvents & EventsForT;
2986 const HWEvents OtherEvents = Other.PendingEvents & EventsForT;
2987 if (!OldEvents.contains(OtherEvents))
2988 StrictDom = true;
2989 PendingEvents |= OtherEvents;
2990
2991 // Merge scores for this counter
2992 const unsigned MyPending = ScoreUBs[T] - ScoreLBs[T];
2993 const unsigned OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
2994 const unsigned NewUB = ScoreLBs[T] + std::max(MyPending, OtherPending);
2995 if (NewUB < ScoreLBs[T])
2996 report_fatal_error("waitcnt score overflow");
2997
2998 MergeInfo &M = MergeInfos[T];
2999 M.OldLB = ScoreLBs[T];
3000 M.OtherLB = Other.ScoreLBs[T];
3001 M.MyShift = NewUB - ScoreUBs[T];
3002 M.OtherShift = NewUB - Other.ScoreUBs[T];
3003
3004 ScoreUBs[T] = NewUB;
3005
3006 if (T == AMDGPU::LOAD_CNT)
3007 StrictDom |= mergeScore(M, LastFlatLoadCnt, Other.LastFlatLoadCnt);
3008
3009 if (T == AMDGPU::X_CNT)
3010 StrictDom |= mergeScore(M, LastXcntOnlyVmem, Other.LastXcntOnlyVmem);
3011
3012 if (T == AMDGPU::DS_CNT) {
3013 StrictDom |= mergeScore(M, LastFlatDsCnt, Other.LastFlatDsCnt);
3014 StrictDom |= mergeScore(M, LastGDS, Other.LastGDS);
3015 }
3016
3017 if (T == AMDGPU::KM_CNT) {
3018 StrictDom |= mergeScore(M, SCCScore, Other.SCCScore);
3019 if (Other.hasPendingEvent(HWEvents::SCC_WRITE)) {
3020 if (!(OldEvents & HWEvents::SCC_WRITE)) {
3021 PendingSCCWrite = Other.PendingSCCWrite;
3022 } else if (PendingSCCWrite != Other.PendingSCCWrite) {
3023 PendingSCCWrite = nullptr;
3024 }
3025 }
3026 }
3027
3028 for (auto &[RegID, Info] : VMem) {
3029 unsigned OtherScore = Other.getVMemScore(RegID, T);
3030 if (T == AMDGPU::VA_VDST_WR) {
3031 // A fence count is usable only when every path carrying the dependency
3032 // has at least that many later ordered XDL writes. Keep this path's
3033 // OrderedXDLCount and rebase each write onto it.
3034 bool MyPending = Info.Scores[T] > M.OldLB;
3035 bool OtherPending = OtherScore > M.OtherLB;
3036 unsigned MyCount = MyPending ? getNumOrderedXDLsAfterWrite(RegID) : 0;
3037 unsigned OtherCount =
3038 OtherPending ? Other.getNumOrderedXDLsAfterWrite(RegID) : 0;
3039 unsigned NewCount = MyPending && OtherPending
3040 ? std::min(MyCount, OtherCount)
3041 : (MyPending ? MyCount : OtherCount);
3042 StrictDom |= MyPending && NewCount < MyCount;
3043 Info.OrderedXDLCountAtWrite = OrderedXDLCount - NewCount;
3044 }
3045 StrictDom |= mergeScore(M, Info.Scores[T], OtherScore);
3046 }
3047
3048 if (isSmemCounter(T)) {
3049 for (auto &[RegID, Info] : SGPRs) {
3050 auto It = Other.SGPRs.find(RegID);
3051 unsigned OtherScore = (It != Other.SGPRs.end()) ? It->second.get(T) : 0;
3052 StrictDom |= mergeScore(M, Info.get(T), OtherScore);
3053 }
3054 }
3055 }
3056
3057 for (auto &[TID, Info] : VMem) {
3058 if (auto It = Other.VMem.find(TID); It != Other.VMem.end()) {
3059 HWEvents NewVGPRContext =
3060 Info.VGPRPendingEvents | It->second.VGPRPendingEvents;
3061 StrictDom |= NewVGPRContext != Info.VGPRPendingEvents;
3062 Info.VGPRPendingEvents = NewVGPRContext;
3063 }
3064 }
3065
3066 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(AMDGPU::AsyncStages::ALL)) {
3067 unsigned SI = AMDGPU::getAsyncStageIndex(S);
3068 StrictDom |= mergeAsyncMarks(S, MergeInfos, Other.AsyncMarks[SI]);
3069 for (auto T : inst_counter_types(Context->MaxCounter))
3070 StrictDom |=
3071 mergeScore(MergeInfos[T], AsyncScore[SI][T], Other.AsyncScore[SI][T]);
3072 }
3073
3074 purgeEmptyTrackingData();
3075 return StrictDom;
3076}
3077
3078static bool isWaitInstr(MachineInstr &Inst) {
3079 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Inst.getOpcode());
3080 return Opcode == AMDGPU::S_WAITCNT ||
3081 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(0).isReg() &&
3082 Inst.getOperand(0).getReg() == AMDGPU::SGPR_NULL) ||
3083 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
3084 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
3085 Opcode == AMDGPU::S_WAITCNT_lds_direct ||
3086 Opcode == AMDGPU::WAIT_ASYNCMARK ||
3087 AMDGPU::counterTypeForInstr(Opcode).has_value();
3088}
3089
3090void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &MBB,
3092 bool ExpertMode) const {
3095 BuildMI(MBB, I, DebugLoc(), TII.get(AMDGPU::S_SETREG_IMM32_B32))
3096 .addImm(ExpertMode ? 2 : 0)
3097 .addImm(EncodedReg);
3098}
3099
3100namespace {
3101// TODO: Remove this work-around after fixing the scheduler.
3102// There are two reasons why vccz might be incorrect; see ST.hasReadVCCZBug()
3103// and ST.partialVCCWritesUpdateVCCZ().
3104// i. VCCZBug: There is a hardware bug on CI/SI where SMRD instruction may
3105// corrupt vccz bit, so when we detect that an instruction may read from
3106// a corrupt vccz bit, we need to:
3107// 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD
3108// operations to complete.
3109// 2. Recompute the correct value of vccz by writing the current value
3110// of vcc back to vcc.
3111// ii. Partial writes to vcc don't update vccz, so we need to recompute the
3112// correct value of vccz by reading vcc and writing it back to vcc.
3113// No waitcnt is needed in this case.
3114class VCCZWorkaround {
3115 const WaitcntBrackets &ScoreBrackets;
3116 const GCNSubtarget &ST;
3117 const SIInstrInfo &TII;
3118 const SIRegisterInfo &TRI;
3119 bool VCCZCorruptionBug = false;
3120 bool VCCZNotUpdatedByPartialWrites = false;
3121 /// vccz could be incorrect at a basic block boundary if a predecessor wrote
3122 /// to vcc and then issued an smem load, so initialize to true.
3123 bool MustRecomputeVCCZ = true;
3124
3125public:
3126 VCCZWorkaround(const WaitcntBrackets &ScoreBrackets, const GCNSubtarget &ST,
3127 const SIInstrInfo &TII, const SIRegisterInfo &TRI)
3128 : ScoreBrackets(ScoreBrackets), ST(ST), TII(TII), TRI(TRI) {
3129 VCCZCorruptionBug = ST.hasReadVCCZBug();
3130 VCCZNotUpdatedByPartialWrites = !ST.partialVCCWritesUpdateVCCZ();
3131 }
3132 /// If \p MI reads vccz and we must recompute it based on MustRecomputeVCCZ,
3133 /// then emit a vccz recompute instruction before \p MI. This needs to be
3134 /// called on every instruction in the basic block because it also tracks the
3135 /// state and updates MustRecomputeVCCZ accordingly. Returns true if it
3136 /// modified the IR.
3137 bool tryRecomputeVCCZ(MachineInstr &MI) {
3138 // No need to run this if neither bug is present.
3139 if (!VCCZCorruptionBug && !VCCZNotUpdatedByPartialWrites)
3140 return false;
3141
3142 // If MI is an SMEM and it can corrupt vccz on this target, then we need
3143 // both to emit a waitcnt and to recompute vccz.
3144 // But we don't actually emit a waitcnt here. This is done in
3145 // generateWaitcntInstBefore() because it tracks all the necessary waitcnt
3146 // state, and can either skip emitting a waitcnt if there is already one in
3147 // the IR, or emit an "optimized" combined waitcnt.
3148 // If this is an smem read, it could complete and clobber vccz at any time.
3149 MustRecomputeVCCZ |= VCCZCorruptionBug && TII.isSMRD(MI);
3150
3151 // If the target partial vcc writes don't update vccz, and MI is such an
3152 // instruction then we must recompute vccz.
3153 // Note: We are using PartiallyWritesToVCCOpt optional to avoid calling
3154 // `definesRegister()` more than needed, because it's not very cheap.
3155 std::optional<bool> PartiallyWritesToVCCOpt;
3156 auto PartiallyWritesToVCC = [](MachineInstr &MI) {
3157 return MI.definesRegister(AMDGPU::VCC_LO, /*TRI=*/nullptr) ||
3158 MI.definesRegister(AMDGPU::VCC_HI, /*TRI=*/nullptr);
3159 };
3160 if (VCCZNotUpdatedByPartialWrites) {
3161 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3162 // If this is a partial VCC write but won't update vccz, then we must
3163 // recompute vccz.
3164 MustRecomputeVCCZ |= *PartiallyWritesToVCCOpt;
3165 }
3166
3167 // If MI is a vcc write with no pending smem, or there is a pending smem
3168 // but the target does not suffer from the vccz corruption bug, then we
3169 // don't need to recompute vccz as this write will recompute it anyway.
3170 if (!ScoreBrackets.hasPendingEvent(HWEvents::SMEM_ACCESS) ||
3171 !VCCZCorruptionBug) {
3172 // Compute PartiallyWritesToVCCOpt if we haven't done so already.
3173 if (!PartiallyWritesToVCCOpt)
3174 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3175 bool FullyWritesToVCC = !*PartiallyWritesToVCCOpt &&
3176 MI.definesRegister(AMDGPU::VCC, /*TRI=*/nullptr);
3177 // If we write to the full vcc or we write partially and the target
3178 // updates vccz on partial writes, then vccz will be updated correctly.
3179 bool UpdatesVCCZ = FullyWritesToVCC || (!VCCZNotUpdatedByPartialWrites &&
3180 *PartiallyWritesToVCCOpt);
3181 if (UpdatesVCCZ)
3182 MustRecomputeVCCZ = false;
3183 }
3184
3185 // If MI is a branch that reads VCCZ then emit a waitcnt and a vccz
3186 // restore instruction if either is needed.
3187 if (SIInstrInfo::isCBranchVCCZRead(MI) && MustRecomputeVCCZ) {
3188 // Recompute the vccz bit. Any time a value is written to vcc, the vccz
3189 // bit is updated, so we can restore the bit by reading the value of vcc
3190 // and then writing it back to the register.
3191 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
3192 TII.get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
3193 TRI.getVCC())
3194 .addReg(TRI.getVCC());
3195 MustRecomputeVCCZ = false;
3196 return true;
3197 }
3198 return false;
3199 }
3200};
3201
3202} // namespace
3203
3204// Generate s_waitcnt instructions where needed.
3205bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
3206 MachineBasicBlock &Block,
3207 WaitcntBrackets &ScoreBrackets) {
3208 bool Modified = false;
3209
3210 LLVM_DEBUG({
3211 dbgs() << "*** Begin Block: ";
3212 Block.printName(dbgs());
3213 ScoreBrackets.dump();
3214 });
3215 VCCZWorkaround VCCZW(ScoreBrackets, ST, TII, TRI);
3216
3217 // Walk over the instructions.
3218 MachineInstr *OldWaitcntInstr = nullptr;
3219
3220 // NOTE: We may append instrs after Inst while iterating.
3221 ScoreBrackets.verify();
3222 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
3223 E = Block.instr_end();
3224 Iter != E; ++Iter) {
3225 MachineInstr &Inst = *Iter;
3226 if (isNonWaitcntMetaInst(Inst))
3227 continue;
3228 // Track pre-existing waitcnts that were added in earlier iterations or by
3229 // the memory legalizer.
3230 if (isWaitInstr(Inst) ||
3231 (IsExpertMode && Inst.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
3232 if (!OldWaitcntInstr)
3233 OldWaitcntInstr = &Inst;
3234 continue;
3235 }
3236
3237 PreheaderFlushFlags FlushFlags;
3238 if (Block.getFirstTerminator() == Inst)
3239 FlushFlags = isPreheaderToFlush(Block, ScoreBrackets);
3240
3241 // Generate an s_waitcnt instruction to be placed before Inst, if needed.
3242 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
3243 FlushFlags);
3244 OldWaitcntInstr = nullptr;
3245
3246 if (Inst.getOpcode() == AMDGPU::ASYNCMARK) {
3247 // Asyncmarks record the current wait state and so should not allow
3248 // waitcnts that occur after them to be merged into waitcnts that occur
3249 // before.
3250 ScoreBrackets.recordAsyncMark(Inst, AMDGPU::asyncStagesFromMaskOperand(
3251 Inst.getOperand(0).getImm()));
3252 continue;
3253 }
3254
3255 if (TII.isSMRD(Inst)) {
3256 for (const MachineMemOperand *Memop : Inst.memoperands()) {
3257 // No need to handle invariant loads when avoiding WAR conflicts, as
3258 // there cannot be a vector store to the same memory location.
3259 if (!Memop->isInvariant()) {
3260 const Value *Ptr = Memop->getValue();
3261 SLoadAddresses.insert(std::pair(Ptr, Inst.getParent()));
3262 }
3263 }
3264 }
3265
3266 updateEventWaitcntAfter(Inst, &ScoreBrackets);
3267
3268 // Note: insertForcedWaitAfter() may add instrs after Iter that need to be
3269 // visited by the loop.
3270 Modified |= insertForcedWaitAfter(Inst, Block, ScoreBrackets);
3271
3272 LLVM_DEBUG({
3273 Inst.print(dbgs());
3274 ScoreBrackets.dump();
3275 });
3276
3277 // If the target suffers from the vccz bugs, this may emit the necessary
3278 // vccz recompute instruction before \p Inst if needed.
3279 Modified |= VCCZW.tryRecomputeVCCZ(Inst);
3280
3281 ScoreBrackets.verify();
3282 }
3283
3284 // Flush counters at the end of the block if needed (for preheaders with no
3285 // terminator).
3286 AMDGPU::Waitcnt Wait;
3287 if (Block.getFirstTerminator() == Block.end()) {
3288 PreheaderFlushFlags FlushFlags = isPreheaderToFlush(Block, ScoreBrackets);
3289 if (FlushFlags.FlushVmCnt) {
3290 if (ScoreBrackets.hasPendingEvent(AMDGPU::LOAD_CNT))
3291 Wait.set(AMDGPU::LOAD_CNT, 0);
3292 if (ScoreBrackets.hasPendingEvent(AMDGPU::SAMPLE_CNT))
3293 Wait.set(AMDGPU::SAMPLE_CNT, 0);
3294 if (ScoreBrackets.hasPendingEvent(AMDGPU::BVH_CNT))
3295 Wait.set(AMDGPU::BVH_CNT, 0);
3296 }
3297 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(AMDGPU::DS_CNT))
3298 Wait.set(AMDGPU::DS_CNT, 0);
3299 }
3300
3301 // Combine or remove any redundant waitcnts at the end of the block.
3302 Modified |= generateWaitcnt(Wait, Block.instr_end(), Block, ScoreBrackets,
3303 OldWaitcntInstr);
3304
3305 LLVM_DEBUG({
3306 dbgs() << "*** End Block: ";
3307 Block.printName(dbgs());
3308 ScoreBrackets.dump();
3309 });
3310
3311 return Modified;
3312}
3313
3314bool SIInsertWaitcnts::removeRedundantSoftXcnts(MachineBasicBlock &Block) {
3315 if (Block.size() <= 1)
3316 return false;
3317 // The Memory Legalizer conservatively inserts a soft xcnt before each
3318 // atomic RMW operation. However, for sequences of back-to-back atomic
3319 // RMWs, only the first s_wait_xcnt insertion is necessary. Optimize away
3320 // the redundant soft xcnts.
3321 bool Modified = false;
3322 // Remember the last atomic with a soft xcnt right before it.
3323 MachineInstr *LastAtomicWithSoftXcnt = nullptr;
3324
3325 for (MachineInstr &MI : drop_begin(Block)) {
3326 // Ignore last atomic if non-LDS VMEM and SMEM.
3327 bool IsLDS = TII.isDS(MI) ||
3328 (TII.isFLAT(MI) && TII.mayAccessLDSThroughFlat(MI, TgSplit));
3329 if (!IsLDS && (MI.mayLoad() ^ MI.mayStore()))
3330 LastAtomicWithSoftXcnt = nullptr;
3331
3332 bool IsAtomicRMW =
3333 SIInstrFlags::isMaybeAtomic(MI) && MI.mayLoad() && MI.mayStore();
3334 MachineInstr &PrevMI = *MI.getPrevNode();
3335 // This is an atomic with a soft xcnt.
3336 if (PrevMI.getOpcode() == AMDGPU::S_WAIT_XCNT_soft && IsAtomicRMW) {
3337 // If we have already found an atomic with a soft xcnt, remove this soft
3338 // xcnt as it's redundant.
3339 if (LastAtomicWithSoftXcnt) {
3340 PrevMI.eraseFromParent();
3341 Modified = true;
3342 }
3343 LastAtomicWithSoftXcnt = &MI;
3344 }
3345 }
3346 return Modified;
3347}
3348
3349// Return flags indicating which counters should be flushed in the preheader.
3350PreheaderFlushFlags
3351SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &MBB,
3352 const WaitcntBrackets &ScoreBrackets) {
3353 auto [Iterator, IsInserted] =
3354 PreheadersToFlush.try_emplace(&MBB, PreheaderFlushFlags());
3355 if (!IsInserted)
3356 return Iterator->second;
3357
3358 MachineBasicBlock *Succ = MBB.getSingleSuccessor();
3359 if (!Succ)
3360 return PreheaderFlushFlags();
3361
3362 MachineLoop *Loop = MLI.getLoopFor(Succ);
3363 if (!Loop)
3364 return PreheaderFlushFlags();
3365
3366 if (Loop->getLoopPreheader() == &MBB) {
3367 Iterator->second = getPreheaderFlushFlags(Loop, ScoreBrackets);
3368 return Iterator->second;
3369 }
3370
3371 return PreheaderFlushFlags();
3372}
3373
3374bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
3376 return TII.mayAccessVMEMThroughFlat(MI);
3377 return SIInstrInfo::isVMEM(MI);
3378}
3379
3380bool SIInsertWaitcnts::isDSRead(const MachineInstr &MI) const {
3381 return SIInstrInfo::isDS(MI) && MI.mayLoad() && !MI.mayStore();
3382}
3383
3384// Check if instruction is a store to LDS that is counted via DSCNT
3385// (where that counter exists).
3386bool SIInsertWaitcnts::mayStoreIncrementingDSCNT(const MachineInstr &MI) const {
3387 return MI.mayStore() && SIInstrInfo::isDS(MI);
3388}
3389
3390// Return flags indicating which counters should be flushed in the preheader of
3391// the given loop. We currently decide to flush in the following situations:
3392// For VMEM (FlushVmCnt):
3393// 1. The loop contains vmem store(s), no vmem load and at least one use of a
3394// vgpr containing a value that is loaded outside of the loop. (Only on
3395// targets with no vscnt counter).
3396// 2. The loop contains vmem load(s), but the loaded values are not used in the
3397// loop, and at least one use of a vgpr containing a value that is loaded
3398// outside of the loop.
3399// For DS (FlushDsCnt, GFX12+ only):
3400// 3. The loop contains no DS reads, and at least one use of a vgpr containing
3401// a value that is DS read outside of the loop.
3402// 4. The loop contains DS read(s), loaded values are not used in the same
3403// iteration but in the next iteration (prefetch pattern), and at least one
3404// use of a vgpr containing a value that is DS read outside of the loop.
3405// Flushing in preheader reduces wait overhead if the wait requirement in
3406// iteration 1 would otherwise be more strict (but unfortunately preheader
3407// flush decision is taken before knowing that).
3408// 5. (Single-block loops only) The loop has DS prefetch reads with flush point
3409// tracking. Some DS reads may be used in the same iteration (creating
3410// "flush points"), but others remain unflushed at the backedge. When a DS
3411// read is consumed in the same iteration, it and all prior reads are
3412// "flushed" (FIFO order). No DS writes are allowed in the loop.
3413// TODO: Find a way to extend to multi-block loops.
3414PreheaderFlushFlags
3415SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
3416 const WaitcntBrackets &Brackets) {
3417 PreheaderFlushFlags Flags;
3418 bool HasVMemLoad = false;
3419 bool HasVMemStore = false;
3420 bool UsesVgprVMEMLoadedOutside = false;
3421 bool UsesVgprDSReadOutside = false;
3422 bool VMemInvalidated = false;
3423 // DS optimization only applies to GFX12+ where DS_CNT is separate.
3424 // Tracking status for "no DS read in loop" or "pure DS prefetch
3425 // (use only in next iteration)".
3426 bool TrackSimpleDSOpt = ST.hasExtendedWaitCounts();
3427 DenseSet<MCRegUnit> VgprUse;
3428 DenseSet<MCRegUnit> VgprDefVMEM;
3429 DenseSet<MCRegUnit> VgprDefDS;
3430
3431 // Track DS reads for prefetch pattern with flush points (single-block only).
3432 // Keeps track of the last DS read (position counted from the top of the loop)
3433 // to each VGPR. Read is considered consumed (and thus needs flushing) if
3434 // the dest register has a use or is overwritten (by any later opertions).
3435 DenseMap<MCRegUnit, unsigned> LastDSReadPositionMap;
3436 unsigned DSReadPosition = 0;
3437 bool IsSingleBlock = ML->getNumBlocks() == 1;
3438 bool TrackDSFlushPoint = ST.hasExtendedWaitCounts() && IsSingleBlock;
3439 unsigned LastDSFlushPosition = 0;
3440
3441 for (MachineBasicBlock *MBB : ML->blocks()) {
3442 for (MachineInstr &MI : *MBB) {
3443 if (isVMEMOrFlatVMEM(MI)) {
3444 HasVMemLoad |= MI.mayLoad();
3445 HasVMemStore |= MI.mayStore();
3446 }
3447 // TODO: Can we relax DSStore check? There may be cases where
3448 // these DS stores are drained prior to the end of MBB (or loop).
3449 if (mayStoreIncrementingDSCNT(MI)) {
3450 // Early exit if none of the optimizations are feasible.
3451 // Otherwise, set tracking status appropriately and continue.
3452 if (VMemInvalidated)
3453 return Flags;
3454 TrackSimpleDSOpt = false;
3455 TrackDSFlushPoint = false;
3456 }
3457 bool IsDSRead = isDSRead(MI);
3458 if (IsDSRead)
3459 ++DSReadPosition;
3460
3461 // Helper: if RU has a pending DS read, update LastDSFlushPosition
3462 auto updateDSReadFlushTracking = [&](MCRegUnit RU) {
3463 if (!TrackDSFlushPoint)
3464 return;
3465 if (auto It = LastDSReadPositionMap.find(RU);
3466 It != LastDSReadPositionMap.end()) {
3467 // RU defined by DSRead is used or overwritten. Need to complete
3468 // the read, if not already implied by a later DSRead (to any RU)
3469 // needing to complete in FIFO order.
3470 LastDSFlushPosition = std::max(LastDSFlushPosition, It->second);
3471 }
3472 };
3473
3474 for (const MachineOperand &Op : MI.all_uses()) {
3475 if (Op.isDebug() || !TRI.isVectorRegister(MRI, Op.getReg()))
3476 continue;
3477 // Vgpr use
3478 for (MCRegUnit RU : TRI.regunits(Op.getReg().asMCReg())) {
3479 // If we find a register that is loaded inside the loop, 1. and 2.
3480 // are invalidated.
3481 if (VgprDefVMEM.contains(RU))
3482 VMemInvalidated = true;
3483
3484 // Check for DS reads used inside the loop
3485 if (VgprDefDS.contains(RU))
3486 TrackSimpleDSOpt = false;
3487
3488 // Early exit if all optimizations are invalidated
3489 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3490 return Flags;
3491
3492 // Check for flush points (DS read used in same iteration)
3493 updateDSReadFlushTracking(RU);
3494
3495 VgprUse.insert(RU);
3496 // Check if this register has a pending VMEM load from outside the
3497 // loop (value loaded outside and used inside).
3498 VMEMID ID = toVMEMID(RU);
3499 if (Brackets.hasPendingVMEM(ID, AMDGPU::LOAD_CNT) ||
3500 Brackets.hasPendingVMEM(ID, AMDGPU::SAMPLE_CNT) ||
3501 Brackets.hasPendingVMEM(ID, AMDGPU::BVH_CNT))
3502 UsesVgprVMEMLoadedOutside = true;
3503 // Check if loaded outside the loop via DS (not VMEM/FLAT).
3504 // Only consider it a DS read if there's no pending VMEM load for
3505 // this register, since FLAT can set both counters.
3506 else if (Brackets.hasPendingVMEM(ID, AMDGPU::DS_CNT))
3507 UsesVgprDSReadOutside = true;
3508 }
3509 }
3510
3511 // VMem load vgpr def
3512 if (isVMEMOrFlatVMEM(MI) && MI.mayLoad()) {
3513 for (const MachineOperand &Op : MI.all_defs()) {
3514 for (MCRegUnit RU : TRI.regunits(Op.getReg().asMCReg())) {
3515 // If we find a register that is loaded inside the loop, 1. and 2.
3516 // are invalidated.
3517 if (VgprUse.contains(RU))
3518 VMemInvalidated = true;
3519 VgprDefVMEM.insert(RU);
3520 }
3521 }
3522 // Early exit if all optimizations are invalidated
3523 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3524 return Flags;
3525 }
3526
3527 // DS read vgpr def
3528 // Note: Unlike VMEM, we DON'T invalidate when VgprUse.contains(RegNo).
3529 // If USE comes before DEF, it's the prefetch pattern (use value from
3530 // previous iteration, read for next iteration). We should still flush
3531 // in preheader so iteration 1 doesn't need to wait inside the loop.
3532 // Only invalidate when DEF comes before USE (same-iteration consumption,
3533 // checked above when processing uses).
3534 if (IsDSRead || TrackDSFlushPoint) {
3535 for (const MachineOperand &Op : MI.all_defs()) {
3536 if (!TRI.isVectorRegister(MRI, Op.getReg()))
3537 continue;
3538 for (MCRegUnit RU : TRI.regunits(Op.getReg().asMCReg())) {
3539 // Check for overwrite of pending DS read (flush point) by any
3540 // instruction
3541 updateDSReadFlushTracking(RU);
3542 if (IsDSRead) {
3543 VgprDefDS.insert(RU);
3544 if (TrackDSFlushPoint)
3545 LastDSReadPositionMap[RU] = DSReadPosition;
3546 }
3547 }
3548 }
3549 }
3550 }
3551 }
3552
3553 // VMEM flush decision
3554 if (!VMemInvalidated && UsesVgprVMEMLoadedOutside &&
3555 ((!ST.hasVscnt() && HasVMemStore && !HasVMemLoad) ||
3556 (HasVMemLoad && ST.hasVmemWriteVgprInOrder())))
3557 Flags.FlushVmCnt = true;
3558
3559 // DS flush decision:
3560 // Simple DS Opt: flush if loop uses DS read values from outside
3561 // and either has no DS reads in the loop, or DS reads whose results
3562 // are not used in the loop.
3563 bool SimpleDSOpt = TrackSimpleDSOpt && UsesVgprDSReadOutside;
3564 // Prefetch with flush points: some DS reads used in same iteration,
3565 // but unflushed reads remain at backedge
3566 bool HasUnflushedDSReads = DSReadPosition > LastDSFlushPosition;
3567 bool DSFlushPointPrefetch =
3568 TrackDSFlushPoint && UsesVgprDSReadOutside && HasUnflushedDSReads;
3569
3570 if (SimpleDSOpt || DSFlushPointPrefetch)
3571 Flags.FlushDsCnt = true;
3572
3573 return Flags;
3574}
3575
3576bool SIInsertWaitcntsLegacy::runOnMachineFunction(MachineFunction &MF) {
3577 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
3578 auto &PDT =
3579 getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
3580 AliasAnalysis *AA = nullptr;
3581 if (auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
3582 AA = &AAR->getAAResults();
3583
3584 return SIInsertWaitcnts(MLI, PDT, AA, MF).run();
3585}
3586
3587PreservedAnalyses
3590 auto &MLI = MFAM.getResult<MachineLoopAnalysis>(MF);
3591 auto &PDT = MFAM.getResult<MachinePostDominatorTreeAnalysis>(MF);
3593 .getManager()
3594 .getCachedResult<AAManager>(MF.getFunction());
3595
3596 if (!SIInsertWaitcnts(MLI, PDT, AA, MF).run())
3597 return PreservedAnalyses::all();
3598
3601 .preserve<AAManager>();
3602}
3603
3604bool SIInsertWaitcnts::run() {
3606
3608
3609 // Initialize hardware limits first, as they're needed by the generators.
3610 Limits = AMDGPU::HardwareLimits(IV);
3611
3612 if (ST.hasExtendedWaitCounts()) {
3613 IsExpertMode = AMDGPU::isExpertSchedulingMode(ST, MF.getFunction());
3614 MaxCounter = IsExpertMode ? AMDGPU::NUM_EXPERT_INST_CNTS
3616 // Initialize WCG per MF. It contains state that depends on MF attributes.
3617 WCG = std::make_unique<WaitcntGeneratorGFX12Plus>(MF, MaxCounter, Limits,
3618 IsExpertMode);
3619 } else {
3620 MaxCounter = AMDGPU::NUM_NORMAL_INST_CNTS;
3621 // Initialize WCG per MF. It contains state that depends on MF attributes.
3622 WCG = std::make_unique<WaitcntGeneratorPreGFX12>(
3623 MF, AMDGPU::NUM_NORMAL_INST_CNTS, Limits);
3624 }
3625
3626 SmemAccessCounter = getCounterFromEvent(HWEvents::SMEM_ACCESS);
3627
3628 bool Modified = false;
3629
3630 MachineBasicBlock &EntryBB = MF.front();
3631
3632 if (!MFI->isEntryFunction() &&
3633 !MF.getFunction().hasFnAttribute(Attribute::Naked)) {
3634 // Wait for any outstanding memory operations that the input registers may
3635 // depend on. We can't track them and it's better to do the wait after the
3636 // costly call sequence.
3637
3638 // TODO: Could insert earlier and schedule more liberally with operations
3639 // that only use caller preserved registers.
3641 while (I != EntryBB.end() && I->isMetaInstruction())
3642 ++I;
3643
3644 if (ST.hasExtendedWaitCounts()) {
3645 BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::S_WAIT_LOADCNT_DSCNT))
3646 .addImm(0);
3648 if (CT == AMDGPU::LOAD_CNT || CT == AMDGPU::DS_CNT ||
3649 CT == AMDGPU::STORE_CNT || CT == AMDGPU::X_CNT ||
3651 continue;
3652
3653 if (!ST.hasImageInsts() &&
3654 (CT == AMDGPU::EXP_CNT || CT == AMDGPU::SAMPLE_CNT ||
3655 CT == AMDGPU::BVH_CNT))
3656 continue;
3657
3658 if (!ST.hasAnyBVHInsts() && CT == AMDGPU::BVH_CNT)
3659 continue;
3660
3661 BuildMI(EntryBB, I, DebugLoc(),
3662 TII.get(instrsForExtendedCounterTypes[CT]))
3663 .addImm(0);
3664 }
3665 if (IsExpertMode) {
3666 unsigned Enc = AMDGPU::DepCtr::encodeFieldVaVdst(0, ST);
3668 BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::S_WAITCNT_DEPCTR))
3669 .addImm(Enc);
3670 }
3671 } else {
3672 BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::S_WAITCNT)).addImm(0);
3673 }
3674
3675 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(this);
3676 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
3677 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
3678
3679 Modified = true;
3680 }
3681
3682 // Keep iterating over the blocks in reverse post order, inserting and
3683 // updating s_waitcnt where needed, until a fix point is reached.
3684 for (auto *MBB : ReversePostOrderTraversal<MachineFunction *>(&MF))
3685 BlockInfos.try_emplace(MBB);
3686
3687 std::unique_ptr<WaitcntBrackets> Brackets;
3688 bool Repeat;
3689 do {
3690 Repeat = false;
3691
3692 for (auto BII = BlockInfos.begin(), BIE = BlockInfos.end(); BII != BIE;
3693 ++BII) {
3694 MachineBasicBlock *MBB = BII->first;
3695 BlockInfo &BI = BII->second;
3696 if (!BI.Dirty)
3697 continue;
3698
3699 if (BI.Incoming) {
3700 if (!Brackets)
3701 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
3702 else
3703 *Brackets = *BI.Incoming;
3704 } else {
3705 if (!Brackets) {
3706 Brackets = std::make_unique<WaitcntBrackets>(this);
3707 } else {
3708 // Reinitialize in-place. N.B. do not do this by assigning from a
3709 // temporary because the WaitcntBrackets class is large and it could
3710 // cause this function to use an unreasonable amount of stack space.
3711 Brackets->~WaitcntBrackets();
3712 new (Brackets.get()) WaitcntBrackets(this);
3713 }
3714 }
3715
3716 if (ST.hasWaitXcnt() && ST.isXNACKEnabled())
3717 Modified |= removeRedundantSoftXcnts(*MBB);
3718 Modified |= insertWaitcntInBlock(MF, *MBB, *Brackets);
3719 BI.Dirty = false;
3720
3721 if (Brackets->hasPendingEvent()) {
3722 BlockInfo *MoveBracketsToSucc = nullptr;
3723 for (MachineBasicBlock *Succ : MBB->successors()) {
3724 auto *SuccBII = BlockInfos.find(Succ);
3725 BlockInfo &SuccBI = SuccBII->second;
3726 if (!SuccBI.Incoming) {
3727 SuccBI.Dirty = true;
3728 if (SuccBII <= BII) {
3729 LLVM_DEBUG(dbgs() << "Repeat on backedge without merge\n");
3730 Repeat = true;
3731 }
3732 if (!MoveBracketsToSucc) {
3733 MoveBracketsToSucc = &SuccBI;
3734 } else {
3735 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
3736 }
3737 } else {
3738 LLVM_DEBUG({
3739 dbgs() << "Try to merge ";
3740 MBB->printName(dbgs());
3741 dbgs() << " into ";
3742 Succ->printName(dbgs());
3743 dbgs() << '\n';
3744 });
3745 if (SuccBI.Incoming->merge(*Brackets)) {
3746 SuccBI.Dirty = true;
3747 if (SuccBII <= BII) {
3748 LLVM_DEBUG(dbgs() << "Repeat on backedge with merge\n");
3749 Repeat = true;
3750 }
3751 }
3752 }
3753 }
3754 if (MoveBracketsToSucc)
3755 MoveBracketsToSucc->Incoming = std::move(Brackets);
3756 }
3757 }
3758 } while (Repeat);
3759
3760 Modified |= WCG->eraseDeferredWaitcnts();
3761
3762 if (ST.hasScalarStores()) {
3763 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
3764 bool HaveScalarStores = false;
3765
3766 for (MachineBasicBlock &MBB : MF) {
3767 for (MachineInstr &MI : MBB) {
3768 if (!HaveScalarStores && TII.isScalarStore(MI))
3769 HaveScalarStores = true;
3770
3771 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
3772 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
3773 EndPgmBlocks.push_back(&MBB);
3774 }
3775 }
3776
3777 if (HaveScalarStores) {
3778 // If scalar writes are used, the cache must be flushed or else the next
3779 // wave to reuse the same scratch memory can be clobbered.
3780 //
3781 // Insert s_dcache_wb at wave termination points if there were any scalar
3782 // stores, and only if the cache hasn't already been flushed. This could
3783 // be improved by looking across blocks for flushes in postdominating
3784 // blocks from the stores but an explicitly requested flush is probably
3785 // very rare.
3786 for (MachineBasicBlock *MBB : EndPgmBlocks) {
3787 bool SeenDCacheWB = false;
3788
3789 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
3790 I != E; ++I) {
3791 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
3792 SeenDCacheWB = true;
3793 else if (TII.isScalarStore(*I))
3794 SeenDCacheWB = false;
3795
3796 // FIXME: It would be better to insert this before a waitcnt if any.
3797 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
3798 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
3799 !SeenDCacheWB) {
3800 Modified = true;
3801 BuildMI(*MBB, I, I->getDebugLoc(), TII.get(AMDGPU::S_DCACHE_WB));
3802 }
3803 }
3804 }
3805 }
3806 }
3807
3808 if (IsExpertMode) {
3809 // Enable expert scheduling on function entry. To satisfy ABI requirements
3810 // and to allow calls between function with different expert scheduling
3811 // settings, disable it around calls and before returns.
3812
3814 while (I != EntryBB.end() && I->isMetaInstruction())
3815 ++I;
3816 setSchedulingMode(EntryBB, I, true);
3817
3818 for (MachineInstr *MI : CallInsts) {
3819 MachineBasicBlock &MBB = *MI->getParent();
3820 setSchedulingMode(MBB, MI, false);
3821 setSchedulingMode(MBB, std::next(MI->getIterator()), true);
3822 }
3823
3824 for (MachineInstr *MI : ReturnInsts)
3825 setSchedulingMode(*MI->getParent(), MI, false);
3826
3827 Modified = true;
3828 }
3829
3830 // Deallocate the VGPRs before previously identified S_ENDPGM instructions.
3831 // This is done in different ways depending on how the VGPRs were allocated
3832 // (i.e. whether we're in dynamic VGPR mode or not).
3833 // Skip deallocation if kernel is waveslot limited vs VGPR limited. A short
3834 // waveslot limited kernel runs slower with the deallocation.
3835 if (!WCG->isOptNone() && MFI->isDynamicVGPREnabled()) {
3836 for (auto [MI, _] : EndPgmInsts) {
3837 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
3838 TII.get(AMDGPU::S_ALLOC_VGPR))
3839 .addImm(0);
3840 Modified = true;
3841 }
3842 } else if (!WCG->isOptNone() &&
3843 ST.getGeneration() >= AMDGPUSubtarget::GFX11 &&
3844 (MF.getFrameInfo().hasCalls() ||
3845 ST.getOccupancyWithNumVGPRs(
3846 TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass),
3847 /*IsDynamicVGPR=*/false) < ST.getMaxWavesPerEU())) {
3848 for (auto [MI, Flag] : EndPgmInsts) {
3849 if (Flag) {
3850 if (ST.requiresNopBeforeDeallocVGPRs()) {
3851 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
3852 TII.get(AMDGPU::S_NOP))
3853 .addImm(0);
3854 }
3855 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
3856 TII.get(AMDGPU::S_SENDMSG))
3858 Modified = true;
3859 }
3860 }
3861 }
3862
3863 if (MFI->isEntryFunction()) {
3864 MachineBasicBlock::iterator InsertPt = EntryBB.begin();
3865
3866 if (ST.hasWaitXcnt() && ST.isXNACKEnabled()) {
3867 // Set REPLAY_MODE (bit 25) in MODE register to enable multi-group XNACK
3868 // replay. This aligns hardware behavior with the compiler's s_wait_xcnt
3869 // insertion logic, which assumes multi-group mode by default.
3870 uint64_t RegEncoding =
3872 BuildMI(EntryBB, InsertPt, DebugLoc(),
3873 TII.get(AMDGPU::S_SETREG_IMM32_B32))
3874 .addImm(1)
3875 .addImm(RegEncoding);
3876 Modified = true;
3877 }
3878
3879 if (ST.hasRequiresInitialUnclausedVmem()) {
3880 // Hardware entrypoints must begin with a specific sequence:
3881 // S_MOV_B64 S[64:65], 0
3882 // V_NOP
3883 // GLOBAL_PREFETCH_B8 V0, S[64:65] SCOPE:SCOPE_SE TH:TH_LOAD_RT
3884 BuildMI(EntryBB, InsertPt, DebugLoc(), TII.get(AMDGPU::S_MOV_B64),
3885 AMDGPU::SGPR64_SGPR65)
3886 .addImm(0);
3887 BuildMI(EntryBB, InsertPt, DebugLoc(), TII.get(AMDGPU::V_NOP_e32));
3888 BuildMI(EntryBB, InsertPt, DebugLoc(),
3889 TII.get(AMDGPU::GLOBAL_PREFETCH_B8_SADDR))
3890 .addReg(AMDGPU::SGPR64_SGPR65)
3891 .addReg(AMDGPU::VGPR0, RegState::Undef)
3892 .addImm(0)
3894 Modified = true;
3895 }
3896 }
3897
3898 return Modified;
3899}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Shared AMDGPU asyncmark stage definitions.
uint64_t uint64_t
uint64_t Imm
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
#define DEBUG_TYPE
const HexagonInstrInfo * TII
static bool isOptNone(const MachineFunction &MF)
#define _
IRTranslator LLVM IR MI
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
static bool isReg(const MCInst &MI, unsigned OpNo)
uint64_t IntrinsicInst * II
#define P(N)
ppc ctr loops verify
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
MachineRegisterInfo & MRI
static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
AMDGPU::HWEvents HWEvents
#define LLVM_DEBUG(...)
Definition Debug.h:119
static Function * getFunction(FunctionType *Ty, const Twine &Name, Module *M)
static const uint32_t IV[8]
Definition blake3_impl.h:83
A manager for alias analyses.
Bit mask of hardware events.
constexpr unsigned size() const
constexpr bool contains(HWEvents Other) const
constexpr bool any() const
unsigned get(InstCounterType T) const
void set(InstCounterType T, unsigned Val)
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:280
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
bool erase(const KeyT &Val)
Definition DenseMap.h:930
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
LLVM_ABI const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
LLVM_ABI void printName(raw_ostream &os, unsigned printNameFlags=PrintNameIr, ModuleSlotTracker *moduleSlotTracker=nullptr) const
Print the basic block's name as:
MachineInstrBundleIterator< MachineInstr > iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Representation of each machine instruction.
mop_range defs()
Returns all explicit operands that are register definitions.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
filtered_mop_range all_defs()
Returns an iterator range over all operands that are (explicit or implicit) register defs.
bool isCall(QueryType Type=AnyInBundle) const
mop_range operands()
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
LLVM_ABI void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
filtered_mop_range all_uses()
Returns an iterator range over all operands that are (explicit or implicit) register uses.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Analysis pass that exposes the MachineLoopInfo for a machine function.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
iterator begin()
Definition MapVector.h:68
iterator find(const KeyT &Key)
Definition MapVector.h:157
iterator end()
Definition MapVector.h:70
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition MapVector.h:119
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:152
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition Register.h:107
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
static bool isCBranchVCCZRead(const MachineInstr &MI)
static bool isDS(const MachineInstr &MI)
static bool isVMEM(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isXcntDrain(const MachineInstr &MI)
True if MI implicitly drains XCNT.
static bool isGWS(const MachineInstr &MI)
static bool isFLATGlobal(const MachineInstr &MI)
static bool isAtomicRet(const MachineInstr &MI)
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static bool isVINTERP(const MachineInstr &MI)
static bool isSBarrierSCCWrite(unsigned Opcode)
static bool isMIMG(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
static bool isLDSDMA(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void push_back(const T &Elt)
Target - Wrapper for Target specific information.
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:183
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
Definition Attributor.h:166
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst)
unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc)
unsigned decodeFieldVaVdst(unsigned Encoded)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
unsigned decodeFieldVmVsrc(unsigned Encoded)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
constexpr AsyncStages asyncStagesFromMaskOperand(int64_t Mask)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
iota_range< InstCounterType > inst_counter_types(InstCounterType MaxCounter)
unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool getHasMatrixScale(unsigned Opc)
AsyncStages
Bit mask of the async stages tracked by the asyncmark / wait_asyncmark intrinsics.
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded)
unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool isTgSplitEnabled(const Function &F)
bool isExpertSchedulingMode(const GCNSubtarget &ST, const Function &F)
Return whether expert scheduling mode is available and enabled for F.
HWEvents getSimplifiedVMEMEventsFor(const MachineInstr &Inst, const SIInstrInfo &TII)
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt)
auto asyncStages(AsyncStages Mask)
unsigned getAsyncStageIndex(AsyncStages Stage)
std::optional< AMDGPU::InstCounterType > counterTypeForInstr(unsigned Opcode)
Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is wa...
constexpr unsigned NumAsyncStages
The number of stages, which is also the number of bits in a stage mask.
HWEvents getEventsFor(const MachineInstr &Inst, const GCNSubtarget &ST, bool IsExpertMode, bool TgSplit)
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt)
unsigned encodeStorecntDscnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
constexpr bool any(E Val)
constexpr bool isMaybeAtomic(const T &...O)
Definition SIDefines.h:325
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
bool empty() const
Definition BasicBlock.h:101
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
@ Wait
Definition Threading.h:60
constexpr auto seq_inclusive(T Begin, T End)
Iterate over an integral type from Begin to End inclusive.
Definition Sequence.h:361
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
auto accumulate(R &&Range, E &&Init)
Wrapper for std::accumulate.
Definition STLExtras.h:1718
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition STLExtras.h:2334
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
LLVM_ABI Printable printRegUnit(MCRegUnit Unit, const TargetRegisterInfo *TRI)
Create Printable object to print register units on a raw_ostream.
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
Definition STLExtras.h:2189
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
char & SIInsertWaitcntsID
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
iterator_range(Container &&) -> iterator_range< llvm::detail::IterOfRange< Container > >
@ Other
Any other memory.
Definition ModRef.h:68
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
@ Increment
Incrementally increasing token ID.
Definition AllocToken.h:26
FunctionPass * createSIInsertWaitcntsPass()
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
#define N
static constexpr ValueType Default
static constexpr uint64_t encode(Fields... Values)
Represents the hardware counter limits for different wait count types.
Instruction set architecture version.