LLVM 24.0.0git
AMDGPUInsertDelayAlu.cpp
Go to the documentation of this file.
1//===- AMDGPUInsertDelayAlu.cpp - Insert s_delay_alu instructions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert s_delay_alu instructions to avoid stalls on GFX11+.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "GCNSubtarget.h"
16#include "SIInstrInfo.h"
18
19using namespace llvm;
20
21#define DEBUG_TYPE "amdgpu-insert-delay-alu"
22
23namespace {
24
25class AMDGPUInsertDelayAlu {
26public:
27 const GCNSubtarget *ST;
28 const SIInstrInfo *SII;
30
31 const TargetSchedModel *SchedModel;
32
33 // Return true if MI waits for all outstanding VALU instructions to complete.
34 static bool instructionWaitsForVALU(const MachineInstr &MI) {
35 // These instruction types wait for VA_VDST==0 before issuing.
39 return true;
40 if (MI.getOpcode() == AMDGPU::S_SENDMSG_RTN_B32 ||
41 MI.getOpcode() == AMDGPU::S_SENDMSG_RTN_B64)
42 return true;
43 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
45 static_cast<unsigned>(MI.getOperand(0).getImm())) == 0)
46 return true;
47 return false;
48 }
49
50 static bool instructionWaitsForSGPRWrites(const MachineInstr &MI) {
51 // These instruction types wait for VA_SDST==0 before issuing.
53 return true;
54
56 for (auto &Op : MI.operands()) {
57 if (Op.isReg())
58 return true;
59 }
60 }
61 return false;
62 }
63
64 // Types of delay that can be encoded in an s_delay_alu instruction.
65 enum DelayType { VALU, TRANS, SALU, OTHER };
66
67 // Get the delay type for a MachineInstr.
68 DelayType getDelayType(const MachineInstr &MI, bool AllowLDSDMA) {
69 // Non-F64 TRANS instructions use a separate delay type.
71 !AMDGPU::isDPMACCInstruction(MI.getOpcode()))
72 return TRANS;
73 // WMMA XDL ops are treated the same as TRANS.
74 if (ST->hasGFX1250Insts() && SII->isXDLWMMA(MI))
75 return TRANS;
76 if (SIInstrInfo::isVALU(MI, AllowLDSDMA))
77 return VALU;
79 return SALU;
80 return OTHER;
81 }
82
83 // Information about the last instruction(s) that wrote to a particular
84 // regunit. In straight-line code there will only be one such instruction, but
85 // when control flow converges we merge the delay information from each path
86 // to represent the union of the worst-case delays of each type.
87 struct DelayInfo {
88 // One larger than the maximum number of (non-TRANS) VALU instructions we
89 // can encode in an s_delay_alu instruction.
90 static constexpr unsigned VALU_MAX = 5;
91
92 // One larger than the maximum number of TRANS instructions we can encode in
93 // an s_delay_alu instruction.
94 static constexpr unsigned TRANS_MAX = 4;
95
96 // One larger than the maximum number of SALU cycles we can encode in an
97 // s_delay_alu instruction.
98 static constexpr unsigned SALU_CYCLES_MAX = 4;
99
100 // If it was written by a (non-TRANS) VALU, remember how many clock cycles
101 // are left until it completes, and how many other (non-TRANS) VALU we have
102 // seen since it was issued.
103 uint8_t VALUCycles = 0;
104 uint8_t VALUNum = VALU_MAX;
105
106 // If it was written by a TRANS, remember how many clock cycles are left
107 // until it completes, and how many other TRANS we have seen since it was
108 // issued.
109 uint8_t TRANSCycles = 0;
110 uint8_t TRANSNum = TRANS_MAX;
111 // Also remember how many other (non-TRANS) VALU we have seen since it was
112 // issued. When an instruction depends on both a prior TRANS and a prior
113 // non-TRANS VALU, this is used to decide whether to encode a wait for just
114 // one or both of them.
115 uint8_t TRANSNumVALU = VALU_MAX;
116
117 // If it was written by an SALU, remember how many clock cycles are left
118 // until it completes.
119 uint8_t SALUCycles = 0;
120
121 // Set when this entry was produced by a data fast-forward producer.
122 bool IsFFProducer = false;
123
124 DelayInfo() = default;
125
126 DelayInfo(DelayType Type, unsigned Cycles, bool IsFFProducer = false)
127 : IsFFProducer(IsFFProducer) {
128 switch (Type) {
129 default:
130 llvm_unreachable("unexpected type");
131 case VALU:
132 VALUCycles = static_cast<uint8_t>(Cycles);
133 VALUNum = 0;
134 break;
135 case TRANS:
136 TRANSCycles = static_cast<uint8_t>(Cycles);
137 TRANSNum = 0;
138 TRANSNumVALU = 0;
139 break;
140 case SALU:
141 // Guard against pseudo-instructions like SI_CALL which are marked as
142 // SALU but with a very high latency.
143 SALUCycles = static_cast<uint8_t>(std::min(Cycles, SALU_CYCLES_MAX));
144 break;
145 }
146 }
147
148 bool operator==(const DelayInfo &RHS) const {
149 return VALUCycles == RHS.VALUCycles && VALUNum == RHS.VALUNum &&
150 TRANSCycles == RHS.TRANSCycles && TRANSNum == RHS.TRANSNum &&
151 TRANSNumVALU == RHS.TRANSNumVALU && SALUCycles == RHS.SALUCycles &&
152 IsFFProducer == RHS.IsFFProducer;
153 }
154
155 bool operator!=(const DelayInfo &RHS) const { return !(*this == RHS); }
156
157 // Merge another DelayInfo into this one, to represent the union of the
158 // worst-case delays of each type.
159 void merge(const DelayInfo &RHS) {
160 VALUCycles = std::max(VALUCycles, RHS.VALUCycles);
161 VALUNum = std::min(VALUNum, RHS.VALUNum);
162 TRANSCycles = std::max(TRANSCycles, RHS.TRANSCycles);
163 TRANSNum = std::min(TRANSNum, RHS.TRANSNum);
164 TRANSNumVALU = std::min(TRANSNumVALU, RHS.TRANSNumVALU);
165 SALUCycles = std::max(SALUCycles, RHS.SALUCycles);
166 IsFFProducer = IsFFProducer && RHS.IsFFProducer;
167 }
168
169 // Update this DelayInfo after issuing an instruction of the specified type.
170 // Cycles is the number of cycles it takes to issue the instruction. Return
171 // true if there is no longer any useful delay info.
172 bool advance(DelayType Type, unsigned Cycles) {
173 bool Erase = true;
174
175 VALUNum += (Type == VALU);
176 if (VALUNum >= VALU_MAX || VALUCycles <= Cycles) {
177 // Forget about the VALU instruction. It was too far back or has
178 // definitely completed by now.
179 VALUNum = VALU_MAX;
180 VALUCycles = 0;
181 } else {
182 VALUCycles -= static_cast<uint8_t>(Cycles);
183 Erase = false;
184 }
185
186 TRANSNum += (Type == TRANS);
187 TRANSNumVALU += (Type == VALU);
188 if (TRANSNum >= TRANS_MAX || TRANSCycles <= Cycles) {
189 // Forget about any TRANS instruction. It was too far back or has
190 // definitely completed by now.
191 TRANSNum = TRANS_MAX;
192 TRANSNumVALU = VALU_MAX;
193 TRANSCycles = 0;
194 } else {
195 TRANSCycles -= static_cast<uint8_t>(Cycles);
196 Erase = false;
197 }
198
199 if (SALUCycles <= Cycles) {
200 // Forget about any SALU instruction. It has definitely completed by
201 // now.
202 SALUCycles = 0;
203 } else {
204 SALUCycles -= static_cast<uint8_t>(Cycles);
205 Erase = false;
206 }
207
208 return Erase;
209 }
210
211#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
212 void dump() const {
213 if (VALUCycles)
214 dbgs() << " VALUCycles=" << (int)VALUCycles;
215 if (VALUNum < VALU_MAX)
216 dbgs() << " VALUNum=" << (int)VALUNum;
217 if (TRANSCycles)
218 dbgs() << " TRANSCycles=" << (int)TRANSCycles;
219 if (TRANSNum < TRANS_MAX)
220 dbgs() << " TRANSNum=" << (int)TRANSNum;
221 if (TRANSNumVALU < VALU_MAX)
222 dbgs() << " TRANSNumVALU=" << (int)TRANSNumVALU;
223 if (SALUCycles)
224 dbgs() << " SALUCycles=" << (int)SALUCycles;
225 }
226#endif
227 };
228
229 // A map from regunits to the delay info for that regunit.
230 struct DelayState : DenseMap<MCRegUnit, DelayInfo> {
231 // Merge another DelayState into this one by merging the delay info for each
232 // regunit.
233 void merge(const DelayState &RHS) {
234 for (const auto &KV : RHS) {
235 iterator It;
236 bool Inserted;
237 std::tie(It, Inserted) = insert(KV);
238 if (!Inserted)
239 It->second.merge(KV.second);
240 }
241 }
242
243 // Advance the delay info for each regunit, erasing any that are no longer
244 // useful.
245 void advance(DelayType Type, unsigned Cycles) {
246 remove_if([&](auto &P) { return P.second.advance(Type, Cycles); });
247 }
248
249 void advanceByVALUNum(unsigned VALUNum) {
250 remove_if([&](auto &P) {
251 return P.second.VALUNum >= VALUNum && P.second.VALUCycles > 0;
252 });
253 }
254
255#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
256 void dump(const TargetRegisterInfo *TRI) const {
257 if (empty()) {
258 dbgs() << " empty\n";
259 return;
260 }
261
262 // Dump DelayInfo for each RegUnit in numerical order.
264 Order.reserve(size());
265 for (const_iterator I = begin(), E = end(); I != E; ++I)
266 Order.push_back(I);
267 llvm::sort(Order, [](const const_iterator &A, const const_iterator &B) {
268 return A->first < B->first;
269 });
270 for (const_iterator I : Order) {
271 dbgs() << " " << printRegUnit(I->first, TRI);
272 I->second.dump();
273 dbgs() << "\n";
274 }
275 }
276#endif
277 };
278
279 // The saved delay state at the end of each basic block.
281
282 // Emit an s_delay_alu instruction if necessary before MI.
283 MachineInstr *emitDelayAlu(MachineInstr &MI, DelayInfo Delay,
284 MachineInstr *LastDelayAlu) {
285 unsigned Imm = 0;
286
287 // Wait for a TRANS instruction.
288 if (Delay.TRANSNum < DelayInfo::TRANS_MAX)
289 Imm |= 4 + Delay.TRANSNum;
290
291 // Wait for a VALU instruction (if it's more recent than any TRANS
292 // instruction that we're also waiting for).
293 if (Delay.VALUNum < DelayInfo::VALU_MAX &&
294 Delay.VALUNum <= Delay.TRANSNumVALU) {
295 if (Imm & 0xf)
296 Imm |= Delay.VALUNum << 7;
297 else
298 Imm |= Delay.VALUNum;
299 }
300
301 // Wait for an SALU instruction.
302 if (Delay.SALUCycles) {
303 assert(Delay.SALUCycles < DelayInfo::SALU_CYCLES_MAX);
304 if (Imm & 0x780) {
305 // We have already encoded a VALU and a TRANS delay. There's no room in
306 // the encoding for an SALU delay as well, so just drop it.
307 } else if (Imm & 0xf) {
308 Imm |= (Delay.SALUCycles + 8) << 7;
309 } else {
310 Imm |= Delay.SALUCycles + 8;
311 }
312 }
313
314 // Don't emit the s_delay_alu instruction if there's nothing to wait for.
315 if (!Imm)
316 return LastDelayAlu;
317
318 // If we only need to wait for one instruction, try encoding it in the last
319 // s_delay_alu that we emitted.
320 if (!(Imm & 0x780) && LastDelayAlu) {
321 unsigned Skip = 0;
322 for (auto I = MachineBasicBlock::instr_iterator(LastDelayAlu),
324 ++I != E;) {
325 if (I->getOpcode() == AMDGPU::S_SET_VGPR_MSB || I->isInlineAsm()) {
326 // It is not deterministic whether the skip count counts
327 // S_SET_VGPR_MSB instructions or not, and we cannot reliably count
328 // the number of instructions in an inline asm, so do not include
329 // either in a skip region.
330 Skip = 6;
331 break;
332 }
333 if (!I->isBundle() && !I->isMetaInstruction())
334 ++Skip;
335 }
336 if (Skip < 6) {
337 MachineOperand &Op = LastDelayAlu->getOperand(0);
338 unsigned LastImm = static_cast<unsigned>(Op.getImm());
339 assert((LastImm & ~0xf) == 0 &&
340 "Remembered an s_delay_alu with no room for another delay!");
341 LastImm |= Imm << 7 | Skip << 4;
342 Op.setImm(LastImm);
343 return nullptr;
344 }
345 }
346
347 auto &MBB = *MI.getParent();
348 MachineInstr *DelayAlu =
349 BuildMI(MBB, MI, DebugLoc(), SII->get(AMDGPU::S_DELAY_ALU)).addImm(Imm);
350 // Remember the s_delay_alu for next time if there is still room in it to
351 // encode another delay.
352 return (Imm & 0x780) ? nullptr : DelayAlu;
353 }
354
355 // Return true if MO is an SGPR def of MI that the hardware can
356 // fast-forward to a matching consumer. These are VALU carry-out / compare
357 // producers whose result can be consumed without an s_delay_alu wait.
358 // V_CMP -> V_CNDMASK
359 // V_ADD/SUB with carry out -> V_ADD/SUB with carry in
360 bool isFastForwardProducer(const MachineInstr &MI, const MachineOperand &MO) {
361 assert((MO.isReg() && MO.isDef()) && "Expected a register definition");
362 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
363 return false;
364 Register Reg = MO.getReg();
365 if (AMDGPU::isSGPR(Reg, TRI)) {
366 switch (MI.getOpcode()) {
367 // VOP3 carry-out producers (explicit VCC/SGPR)
368 case AMDGPU::V_ADD_CO_U32_e64:
369 case AMDGPU::V_SUB_CO_U32_e64:
370 case AMDGPU::V_SUBREV_CO_U32_e64:
371 case AMDGPU::V_ADDC_U32_e64:
372 case AMDGPU::V_SUBB_U32_e64:
373 case AMDGPU::V_SUBBREV_U32_e64:
374 return true;
375 default:
376 if (MI.isCompare())
377 return true;
378 }
379 }
380 return false;
381 }
382
383 // Return true if operand OpNo (MO) of MI is an SGPR use that can receive
384 // fast-forwarded data from a matching producer, letting the s_delay_alu
385 // wait be skipped. These are VALU carry-in / condition-mask consumers that
386 // read the forwarded value
387 // V_CMP -> V_CNDMASK
388 // V_ADD/SUB with carry out -> V_ADD/SUB with carry in
389 bool isFastForwardConsumer(const MachineInstr &MI, const MachineOperand &MO,
390 unsigned OpNo) {
391 assert((MO.isReg() && MO.isUse()) && "Expected a register use");
392 if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
393 return false;
394 Register Reg = MO.getReg();
395 unsigned MIOpCode = MI.getOpcode();
396
397 if (AMDGPU::isSGPR(Reg, TRI)) {
398 switch (MIOpCode) {
399 // VOP3 explicit carry-in / condition-mask consumers
400 case AMDGPU::V_ADDC_U32_e64:
401 case AMDGPU::V_SUBB_U32_e64:
402 case AMDGPU::V_SUBBREV_U32_e64:
403 case AMDGPU::V_CNDMASK_B32_e64:
404 case AMDGPU::V_CNDMASK_B16_fake16_e64:
405 case AMDGPU::V_CNDMASK_B16_t16_e64:
406 int Src2Idx =
407 AMDGPU::getNamedOperandIdx(MIOpCode, AMDGPU::OpName::src2);
408 assert(Src2Idx >= 0 && "Unexpected source index");
409 return OpNo == static_cast<unsigned>(Src2Idx);
410 }
411 }
412 return false;
413 }
414
416 DelayState State;
417 for (auto *Pred : MBB.predecessors())
418 State.merge(BlockState[Pred]);
419
420 LLVM_DEBUG(dbgs() << " State at start of " << printMBBReference(MBB)
421 << "\n";
422 State.dump(TRI););
423
424 bool Changed = false;
425 MachineInstr *LastDelayAlu = nullptr;
426
427 // FIXME: 0 is a valid register unit.
428 MCRegUnit LastSGPRFromVALU = static_cast<MCRegUnit>(0);
429
430 // Destination of the preceding WMMA, for C-reuse detection.
431 Register PrevWMMAVDst;
432
433 // Iterate over the contents of bundles, but don't emit any instructions
434 // inside a bundle.
435 for (auto &MI : MBB.instrs()) {
436 if (MI.isBundle() || MI.isMetaInstruction())
437 continue;
438
439 // Ignore some more instructions that do not generate any code.
440 switch (MI.getOpcode()) {
441 case AMDGPU::SI_RETURN_TO_EPILOG:
442 continue;
443 }
444
445 // LDSDMA is VALU-tagged but only behaves like VALU for operand-use delay
446 // checks (e.g. v_readfirstlane -> tensor_load_to_lds). It must not
447 // publish or advance VALU delay state on its defs.
448 DelayType ProducerType = getDelayType(MI, /*AllowLDSDMA=*/false);
449 DelayType ConsumerType = getDelayType(MI, /*AllowLDSDMA=*/true);
450
451 if (instructionWaitsForSGPRWrites(MI)) {
452 auto It = State.find(LastSGPRFromVALU);
453 if (It != State.end()) {
454 DelayInfo Info = It->getSecond();
455 State.advanceByVALUNum(Info.VALUNum);
456 // FIXME: 0 is a valid register unit.
457 LastSGPRFromVALU = static_cast<MCRegUnit>(0);
458 }
459 }
460
461 if (instructionWaitsForVALU(MI)) {
462 // Forget about all outstanding VALU delays.
463 // TODO: This is overkill since it also forgets about SALU delays.
464 State = DelayState();
465 } else if (ConsumerType != OTHER) {
466 DelayInfo Delay;
467 // C-reuse: back-to-back WMMAs into the same C register forward the
468 // accumulator in place, so the tied srcC read has no dependency. WMMA
469 // implies GFX11+, so no explicit subtarget check is needed.
470 bool IsWMMACReuse =
471 PrevWMMAVDst.isValid() && (SII->isWMMA(MI) || SII->isSWMMAC(MI));
472 // TODO: Scan implicit uses too?
473 for (const auto &Op : MI.explicit_uses()) {
474 if (Op.isReg()) {
475 // One of the operands of the writelane is also the output operand.
476 // This creates the insertion of redundant delays. Hence, we have to
477 // ignore this operand.
478 if (MI.getOpcode() == AMDGPU::V_WRITELANE_B32 && Op.isTied())
479 continue;
480 // Skip the tied srcC of a C-reuse edge.
481 if (IsWMMACReuse && Op.isTied() && Op.getReg() == PrevWMMAVDst)
482 continue;
483
484 Register Reg = Op.getReg();
485 unsigned OperandNo = MI.getOperandNo(&Op);
486
487 // Suppress the delay for SGPR operands when both producer and
488 // consumer are in the hardware valu data fast-forward set.
489 if (isFastForwardConsumer(MI, Op, OperandNo) &&
490 llvm::all_of(TRI->regunits(Reg), [&](MCRegUnit Unit) {
491 auto It = State.find(Unit);
492 if (It != State.end())
493 return It->second.IsFFProducer;
494 return true; // no wait for this regunit if it has no delay
495 })) {
496 continue;
497 }
498
499 for (MCRegUnit Unit : TRI->regunits(Op.getReg())) {
500 auto It = State.find(Unit);
501 if (It != State.end()) {
502 Delay.merge(It->second);
503 State.erase(Unit);
504 }
505 }
506 }
507 }
508
509 if (ProducerType == VALU) {
510 for (const auto &Op : MI.defs()) {
511 Register Reg = Op.getReg();
512 if (AMDGPU::isSGPR(Reg, TRI)) {
513 LastSGPRFromVALU = *TRI->regunits(Reg).begin();
514 break;
515 }
516 }
517 }
518
519 if (Emit && !MI.isBundledWithPred()) {
520 // TODO: For VALU->SALU delays should we use s_delay_alu or s_nop or
521 // just ignore them?
522 LastDelayAlu = emitDelayAlu(MI, Delay, LastDelayAlu);
523 }
524 }
525
526 if (ProducerType != OTHER) {
527 // TODO: Scan implicit defs too?
528 for (const auto &Op : MI.defs()) {
529 unsigned Latency = SchedModel->computeOperandLatency(
530 &MI, Op.getOperandNo(), nullptr, 0);
531 for (MCRegUnit Unit : TRI->regunits(Op.getReg()))
532 State[Unit] =
533 DelayInfo{ProducerType, Latency, isFastForwardProducer(MI, Op)};
534 }
535 }
536
537 // Advance by the number of cycles it takes to issue this instruction.
538 // TODO: Use a more advanced model that accounts for instructions that
539 // take multiple cycles to issue on a particular pipeline.
540 unsigned Cycles = SIInstrInfo::getNumWaitStates(MI);
541 // TODO: In wave64 mode, double the number of cycles for VALU and VMEM
542 // instructions on the assumption that they will usually have to be issued
543 // twice?
544 State.advance(ProducerType, Cycles);
545
546 if (MI.getOpcode() == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay) {
547 // The load clears prior delays, but its MC expansion ends in an EXEC
548 // restore. Model that scalar write without an EXEC def in the MIR.
549 DelayInfo RestoreDelay(
550 SALU, SchedModel->computeInstrLatency(AMDGPU::S_MOV_B32));
551 if (!RestoreDelay.advance(SALU, 1)) {
552 for (MCRegUnit Unit : TRI->regunits(AMDGPU::EXEC_LO))
553 State[Unit] = RestoreDelay;
554 }
555 // A single MIR instruction expands to four ISA instructions.
556 LastDelayAlu = nullptr;
557 }
558
559 // Track the preceding WMMA's dst for C-reuse; reset on anything else.
560 if (SII->isWMMA(MI) || SII->isSWMMAC(MI)) {
561 const MachineOperand *VDst =
562 SII->getNamedOperand(MI, AMDGPU::OpName::vdst);
563 PrevWMMAVDst = VDst ? VDst->getReg() : Register();
564 } else {
565 PrevWMMAVDst = Register();
566 }
567
568 LLVM_DEBUG(dbgs() << " State after " << MI; State.dump(TRI););
569 }
570
571 if (Emit) {
572 assert(State == BlockState[&MBB] &&
573 "Basic block state should not have changed on final pass!");
574 } else if (DelayState &BS = BlockState[&MBB]; State != BS) {
575 BS = std::move(State);
576 Changed = true;
577 }
578 return Changed;
579 }
580
581 bool run(MachineFunction &MF) {
582 LLVM_DEBUG(dbgs() << "AMDGPUInsertDelayAlu running on " << MF.getName()
583 << "\n");
584
585 ST = &MF.getSubtarget<GCNSubtarget>();
586 if (!ST->hasDelayAlu())
587 return false;
588
590
591 if (MFI.getMaxWavesPerEU() == 1)
592 return false;
593
594 SII = ST->getInstrInfo();
595 TRI = ST->getRegisterInfo();
596 SchedModel = &SII->getSchedModel();
597
598 // Calculate the delay state for each basic block, iterating until we reach
599 // a fixed point.
601 for (auto &MBB : reverse(MF))
602 WorkList.insert(&MBB);
603 while (!WorkList.empty()) {
604 auto &MBB = *WorkList.pop_back_val();
605 bool Changed = runOnMachineBasicBlock(MBB, false);
606 if (Changed)
607 WorkList.insert_range(MBB.successors());
608 }
609
610 LLVM_DEBUG(dbgs() << "Final pass over all BBs\n");
611
612 // Make one last pass over all basic blocks to emit s_delay_alu
613 // instructions.
614 bool Changed = false;
615 for (auto &MBB : MF)
617 return Changed;
618 }
619};
620
621class AMDGPUInsertDelayAluLegacy : public MachineFunctionPass {
622public:
623 static char ID;
624
625 AMDGPUInsertDelayAluLegacy() : MachineFunctionPass(ID) {}
626
627 void getAnalysisUsage(AnalysisUsage &AU) const override {
628 AU.setPreservesCFG();
630 }
631
632 bool runOnMachineFunction(MachineFunction &MF) override {
633 if (skipFunction(MF.getFunction()))
634 return false;
635 AMDGPUInsertDelayAlu Impl;
636 return Impl.run(MF);
637 }
638};
639} // namespace
640
644 if (!AMDGPUInsertDelayAlu().run(MF))
645 return PreservedAnalyses::all();
647 PA.preserveSet<CFGAnalyses>();
648 return PA;
649} // end namespace llvm
650
651char AMDGPUInsertDelayAluLegacy::ID = 0;
652
653char &llvm::AMDGPUInsertDelayAluID = AMDGPUInsertDelayAluLegacy::ID;
654
655INITIALIZE_PASS(AMDGPUInsertDelayAluLegacy, DEBUG_TYPE,
656 "AMDGPU Insert Delay ALU", false, false)
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t Imm
MachineBasicBlock & MBB
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
#define DEBUG_TYPE
IRTranslator LLVM IR MI
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static bool runOnMachineBasicBlock(MachineBasicBlock &MBB, MachineInstr *FirstNextMBBInstr, function_ref< MachineBranchProbabilityInfo &()> GetMBPI)
runOnMachineBasicBlock - Fill in delay slots for the given basic block.
#define P(N)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
Interface definition for SIInstrInfo.
#define LLVM_DEBUG(...)
Definition Debug.h:119
Value * RHS
Represent the analysis usage information of a pass.
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:280
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
Instructions::iterator instr_iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
bool isXDLWMMA(const MachineInstr &MI) const
static bool isSALU(const MachineInstr &MI)
static bool isSWMMAC(const MachineInstr &MI)
const TargetSchedModel & getSchedModel() const
static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA)
static bool isTRANS(const MachineInstr &MI)
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
static bool isWMMA(const MachineInstr &MI)
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, AMDGPU::OpName OperandName) const
Returns the operand named Op.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
A vector that has set insertion semantics.
Definition SetVector.h:57
void insert_range(Range &&R)
Definition SetVector.h:182
bool empty() const
Determine if the SetVector is empty or not.
Definition SetVector.h:100
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
value_type pop_back_val()
Definition SetVector.h:285
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Provide an instruction scheduling machine model to CodeGen passes.
LLVM_ABI unsigned computeOperandLatency(const MachineInstr *DefMI, unsigned DefOperIdx, const MachineInstr *UseMI, unsigned UseOperIdx) const
Compute operand latency based on the available machine model.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned decodeFieldVaVdst(unsigned Encoded)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
bool isDPMACCInstruction(unsigned Opc)
constexpr bool isFLAT(const T &...O)
Definition SIDefines.h:286
constexpr bool isBuffer(const T &...O)
Definition SIDefines.h:267
constexpr bool isSMRD(const T &...O)
Definition SIDefines.h:271
constexpr bool isMIMG(const T &...O)
Definition SIDefines.h:274
constexpr bool isEXP(const T &...O)
Definition SIDefines.h:283
constexpr bool isDS(const T &...O)
Definition SIDefines.h:289
constexpr bool isSALU(const T &...O)
Definition SIDefines.h:209
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
Definition APInt.h:2139
LLVM_ABI Printable printRegUnit(MCRegUnit Unit, const TargetRegisterInfo *TRI)
Create Printable object to print register units on a raw_ostream.
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
char & AMDGPUInsertDelayAluID
auto remove_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1800
DWARFExpression::Operation Op
LLVM_ABI Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
PreservedAnalyses run(MachineFunction &F, MachineFunctionAnalysisManager &MFAM)