LLVM 24.0.0git
SIInstrInfo.cpp
Go to the documentation of this file.
1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPUMachineInstrs.h"
19#include "GCNHazardRecognizer.h"
20#include "GCNSubtarget.h"
23#include "llvm/ADT/STLExtras.h"
34#include "llvm/IR/IntrinsicsAMDGPU.h"
35#include "llvm/MC/MCContext.h"
38#include <tuple>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "si-instr-info"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#include "AMDGPUGenInstrInfo.inc"
46
47namespace llvm::AMDGPU {
48#define GET_ImageDimIntrinsicTable_IMPL
49#define GET_RsrcIntrinsics_IMPL
50#define GET_BufferIntrinsicsTable_IMPL
51#define GET_GFX1250BlockingCyclesTable_DECL
52#define GET_GFX1250BlockingCyclesTable_IMPL
53
58
59#include "AMDGPUGenSearchableTables.inc"
60} // namespace llvm::AMDGPU
61
63 : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
64 AMDGPU::ADJCALLSTACKDOWN),
65 RI(ST), ST(ST) {
66 SchedModel.init(&ST);
67}
68
69//===----------------------------------------------------------------------===//
70// TargetInstrInfo callbacks
71//===----------------------------------------------------------------------===//
72
73static unsigned getNumOperandsNoGlue(SDNode *Node) {
74 unsigned N = Node->getNumOperands();
75 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
76 --N;
77 return N;
78}
79
80/// Returns true if both nodes have the same value for the given
81/// operand \p Op, or if both nodes do not have this operand.
83 AMDGPU::OpName OpName) {
84 unsigned Opc0 = N0->getMachineOpcode();
85 unsigned Opc1 = N1->getMachineOpcode();
86
87 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
88 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
89
90 if (Op0Idx == -1 && Op1Idx == -1)
91 return true;
92
93
94 if ((Op0Idx == -1 && Op1Idx != -1) ||
95 (Op1Idx == -1 && Op0Idx != -1))
96 return false;
97
98 // getNamedOperandIdx returns the index for the MachineInstr's operands,
99 // which includes the result as the first operand. We are indexing into the
100 // MachineSDNode's operands, so we need to skip the result operand to get
101 // the real index.
102 --Op0Idx;
103 --Op1Idx;
104
105 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
106}
107
108static bool canRemat(const MachineInstr &MI) {
109
113 return true;
114
115 if (SIInstrInfo::isSMRD(MI)) {
116 return !MI.memoperands_empty() &&
117 llvm::all_of(MI.memoperands(), [](const MachineMemOperand *MMO) {
118 return MMO->isLoad() && MMO->isInvariant();
119 });
120 }
121
122 return false;
123}
124
125// Split relocation flags for 64-bit global-address materialization into the
126// flags for the lo and hi halves.
127static std::pair<unsigned, unsigned>
129 const unsigned Flags = SrcOp.getTargetFlags();
130
131 // Infer the relocation type from the existing flags on the global operand.
132 // The relocation type should have been determined earlier in the pipeline.
133 switch (Flags & SIInstrInfo::MO_MASK) {
136 // For 64-bit GOT-relative, use the 64-bit relocation.
137 const unsigned GotFlags =
138 (Flags & ~SIInstrInfo::MO_MASK) | SIInstrInfo::MO_GOTPCREL64;
139 return {GotFlags, GotFlags};
140 }
141 default:
142 break;
143 }
144
145 std::optional<unsigned> LoFlags =
147 std::optional<unsigned> HiFlags =
149 if (!LoFlags || !HiFlags)
150 llvm_unreachable("unknown relocation type for global address");
151 return {*LoFlags, *HiFlags};
152}
153
155 const MachineInstr &MI) const {
156
157 if (canRemat(MI)) {
158 // Normally VALU use of exec would block the rematerialization, but that
159 // is OK in this case to have an implicit exec read as all VALU do.
160 // We really want all of the generic logic for this except for this.
161
162 // Another potential implicit use is mode register. The core logic of
163 // the RA will not attempt rematerialization if mode is set anywhere
164 // in the function, otherwise it is safe since mode is not changed.
165
166 // There is difference to generic method which does not allow
167 // rematerialization if there are virtual register uses. We allow this,
168 // therefore this method includes SOP instructions as well.
169 if (!MI.hasImplicitDef() &&
170 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
171 !MI.mayRaiseFPException())
172 return true;
173 }
174
175 // Everything below copied from TargetInstrInfo::isReMaterializableImpl. The
176 // only difference is that we allow operations that perform read-modify-write
177 // on sub-registers.
178
179 // Remat clients assume operand 0 is the defined register.
180 if (!MI.getNumOperands() || !MI.getOperand(0).isReg())
181 return false;
182 Register DefReg = MI.getOperand(0).getReg();
183
184 const MachineFunction &MF = *MI.getMF();
185
186 // A load from a fixed stack slot can be rematerialized. This may be
187 // redundant with subsequent checks, but it's target-independent,
188 // simple, and a common case.
189 int FrameIdx = 0;
190 if (isLoadFromStackSlot(MI, FrameIdx) &&
192 return true;
193
194 // Avoid instructions obviously unsafe for remat.
195 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
196 MI.hasUnmodeledSideEffects())
197 return false;
198
199 // Don't remat inline asm. We have no idea how expensive it is
200 // even if it's side effect free.
201 if (MI.isInlineAsm())
202 return false;
203
204 // Avoid instructions which load from potentially varying memory.
205 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
206 return false;
207
208 const MachineRegisterInfo &MRI = MF.getRegInfo();
209
210 // If any of the registers accessed are non-constant, conservatively assume
211 // the instruction is not rematerializable.
212 for (const MachineOperand &MO : MI.operands()) {
213 if (!MO.isReg())
214 continue;
215 Register Reg = MO.getReg();
216 if (Reg == 0)
217 continue;
218
219 // Check for a well-behaved physical register.
220 if (Reg.isPhysical()) {
221 if (MO.isUse()) {
222 // If the physreg has no defs anywhere, it's just an ambient register
223 // and we can freely move its uses. Alternatively, if it's allocatable,
224 // it could get allocated to something with a def during allocation.
225 if (!MRI.isConstantPhysReg(Reg))
226 return false;
227 } else {
228 // A physreg def. We can't remat it.
229 return false;
230 }
231 continue;
232 }
233
234 // Only allow one virtual-register def. There may be multiple defs of the
235 // same virtual register, though.
236 if (MO.isDef() && Reg != DefReg)
237 return false;
238 }
239
240 return true;
241}
242
244 switch (Opcode) {
245 // v_subrev_u16 (gfx9)
246 case AMDGPU::V_SUBREV_U16_e32:
247 case AMDGPU::V_SUBREV_U16_e64:
248 // v_subrev_u32 (gfx9) / v_subrev_nc_u32 (gfx10+)
249 case AMDGPU::V_SUBREV_U32_e32:
250 case AMDGPU::V_SUBREV_U32_e64:
251 // v_subrev_co_u32
252 case AMDGPU::V_SUBREV_CO_U32_e32:
253 case AMDGPU::V_SUBREV_CO_U32_e64:
254 // v_subbrev_u32 (gfx9) / v_subrev_co_ci_u32 (gfx10+)
255 case AMDGPU::V_SUBBREV_U32_e32:
256 case AMDGPU::V_SUBBREV_U32_e64:
257 return true;
258 // REV shift opcodes worked this way before GFX11, verified on hardware
259 case AMDGPU::V_ASHRREV_I16_e32:
260 case AMDGPU::V_ASHRREV_I16_e64:
261 case AMDGPU::V_ASHRREV_I32_e32:
262 case AMDGPU::V_ASHRREV_I32_e64:
263 case AMDGPU::V_ASHRREV_I64_e64:
264 case AMDGPU::V_LSHLREV_B16_e32:
265 case AMDGPU::V_LSHLREV_B16_e64:
266 case AMDGPU::V_LSHLREV_B32_e32:
267 case AMDGPU::V_LSHLREV_B32_e64:
268 case AMDGPU::V_LSHLREV_B64_e64:
269 case AMDGPU::V_LSHRREV_B16_e32:
270 case AMDGPU::V_LSHRREV_B16_e64:
271 case AMDGPU::V_LSHRREV_B32_e32:
272 case AMDGPU::V_LSHRREV_B32_e64:
273 case AMDGPU::V_LSHRREV_B64_e64:
274 return !ST.hasGFX11Insts();
275 default:
276 return false;
277 }
278}
279
280// Returns true if the result of a VALU instruction depends on exec.
281bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
282 assert(isVALU(MI, /*AllowLDSDMA=*/true));
283
284 // If it is convergent it depends on EXEC.
285 if (MI.isConvergent())
286 return true;
287
288 // If it defines an SGPR it depends on EXEC, unless it's dead.
289 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
290 for (const MachineOperand &Def : MI.defs()) {
291 if (Def.isDead())
292 continue;
293
294 Register Reg = Def.getReg();
295 if (Reg && RI.isSGPRReg(MRI, Reg))
296 return true;
297 }
298
299 return false;
300}
301
302bool SIInstrInfo::isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const {
303 const MachineOperand &MO = MI.getOperand(OpIdx);
304 // Any implicit use of exec by VALU is not a real register read.
305 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
306 isVALU(MI, /*AllowLDSDMA=*/true) && !resultDependsOnExec(MI);
307}
308
310 MachineBasicBlock *SuccToSinkTo,
311 MachineCycleInfo *CI) const {
312 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).
313 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
314 return true;
315
316 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
317 // Check if sinking of MI would create temporal divergent use.
318 for (auto Op : MI.uses()) {
319 if (Op.isReg() && Op.getReg().isVirtual() &&
320 RI.isSGPRClass(MRI.getRegClass(Op.getReg()))) {
321 MachineInstr *SgprDef = MRI.getVRegDef(Op.getReg());
322 if (!SgprDef)
323 continue;
324
325 // SgprDef defined inside cycle
326 CycleRef FromCycle = CI->getCycle(SgprDef->getParent());
327 if (!FromCycle)
328 continue;
329
330 CycleRef ToCycle = CI->getCycle(SuccToSinkTo);
331 // Check if there is a FromCycle that contains SgprDef's basic block but
332 // does not contain SuccToSinkTo and also has divergent exit condition.
333 while (FromCycle && !(ToCycle && CI->contains(FromCycle, ToCycle))) {
335 CI->getExitingBlocks(FromCycle, ExitingBlocks);
336
337 // FromCycle has divergent exit condition.
338 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
339 if (hasDivergentBranch(ExitingBlock))
340 return false;
341 }
342
343 FromCycle = CI->getParentCycle(FromCycle);
344 }
345 }
346 }
347
348 return true;
349}
350
352 int64_t &Offset0,
353 int64_t &Offset1) const {
354 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
355 return false;
356
357 unsigned Opc0 = Load0->getMachineOpcode();
358 unsigned Opc1 = Load1->getMachineOpcode();
359
360 // Make sure both are actually loads.
361 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
362 return false;
363
364 // A mayLoad instruction without a def is not a load. Likely a prefetch.
365 if (!get(Opc0).getNumDefs() || !get(Opc1).getNumDefs())
366 return false;
367
368 if (isDS(Opc0) && isDS(Opc1)) {
369
370 // FIXME: Handle this case:
371 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
372 return false;
373
374 // Check base reg.
375 if (Load0->getOperand(0) != Load1->getOperand(0))
376 return false;
377
378 // Skip read2 / write2 variants for simplicity.
379 // TODO: We should report true if the used offsets are adjacent (excluded
380 // st64 versions).
381 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
382 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
383 if (Offset0Idx == -1 || Offset1Idx == -1)
384 return false;
385
386 // XXX - be careful of dataless loads
387 // getNamedOperandIdx returns the index for MachineInstrs. Since they
388 // include the output in the operand list, but SDNodes don't, we need to
389 // subtract the index by one.
390 Offset0Idx -= get(Opc0).NumDefs;
391 Offset1Idx -= get(Opc1).NumDefs;
392 Offset0 = Load0->getConstantOperandVal(Offset0Idx);
393 Offset1 = Load1->getConstantOperandVal(Offset1Idx);
394 return true;
395 }
396
397 if (isSMRD(Opc0) && isSMRD(Opc1)) {
398 // Skip time and cache invalidation instructions.
399 if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) ||
400 !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase))
401 return false;
402
403 unsigned NumOps = getNumOperandsNoGlue(Load0);
404 if (NumOps != getNumOperandsNoGlue(Load1))
405 return false;
406
407 // Check base reg.
408 if (Load0->getOperand(0) != Load1->getOperand(0))
409 return false;
410
411 // Match register offsets, if both register and immediate offsets present.
412 assert(NumOps == 4 || NumOps == 5);
413 if (NumOps == 5 && Load0->getOperand(1) != Load1->getOperand(1))
414 return false;
415
416 const ConstantSDNode *Load0Offset =
418 const ConstantSDNode *Load1Offset =
420
421 if (!Load0Offset || !Load1Offset)
422 return false;
423
424 Offset0 = Load0Offset->getZExtValue();
425 Offset1 = Load1Offset->getZExtValue();
426 return true;
427 }
428
429 // MUBUF and MTBUF can access the same addresses.
430 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
431
432 // MUBUF and MTBUF have vaddr at different indices.
433 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
434 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
435 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
436 return false;
437
438 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
439 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
440
441 if (OffIdx0 == -1 || OffIdx1 == -1)
442 return false;
443
444 // getNamedOperandIdx returns the index for MachineInstrs. Since they
445 // include the output in the operand list, but SDNodes don't, we need to
446 // subtract the index by one.
447 OffIdx0 -= get(Opc0).NumDefs;
448 OffIdx1 -= get(Opc1).NumDefs;
449
450 SDValue Off0 = Load0->getOperand(OffIdx0);
451 SDValue Off1 = Load1->getOperand(OffIdx1);
452
453 // The offset might be a FrameIndexSDNode.
454 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
455 return false;
456
457 Offset0 = Off0->getAsZExtVal();
458 Offset1 = Off1->getAsZExtVal();
459 return true;
460 }
461
462 return false;
463}
464
465static bool isStride64(unsigned Opc) {
466 switch (Opc) {
467 case AMDGPU::DS_READ2ST64_B32:
468 case AMDGPU::DS_READ2ST64_B64:
469 case AMDGPU::DS_WRITE2ST64_B32:
470 case AMDGPU::DS_WRITE2ST64_B64:
471 return true;
472 default:
473 return false;
474 }
475}
476
479 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
480 if (!LdSt.mayLoadOrStore())
481 return false;
482
483 unsigned Opc = LdSt.getOpcode();
484 OffsetIsScalable = false;
485 const MachineOperand *BaseOp, *OffsetOp;
486 int DataOpIdx;
487
488 if (isDS(LdSt)) {
489 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
490 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
491 if (OffsetOp) {
492 // Normal, single offset LDS instruction.
493 if (!BaseOp) {
494 // DS_CONSUME/DS_APPEND use M0 for the base address.
495 // TODO: find the implicit use operand for M0 and use that as BaseOp?
496 return false;
497 }
498 BaseOps.push_back(BaseOp);
499 Offset = OffsetOp->getImm();
500 // Get appropriate operand, and compute width accordingly.
501 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
502 if (DataOpIdx == -1)
503 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
504 if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
505 Width = LocationSize::precise(64);
506 else
507 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
508 } else {
509 // The 2 offset instructions use offset0 and offset1 instead. We can treat
510 // these as a load with a single offset if the 2 offsets are consecutive.
511 // We will use this for some partially aligned loads.
512 const MachineOperand *Offset0Op =
513 getNamedOperand(LdSt, AMDGPU::OpName::offset0);
514 const MachineOperand *Offset1Op =
515 getNamedOperand(LdSt, AMDGPU::OpName::offset1);
516
517 unsigned Offset0 = Offset0Op->getImm() & 0xff;
518 unsigned Offset1 = Offset1Op->getImm() & 0xff;
519 if (Offset0 + 1 != Offset1)
520 return false;
521
522 // Each of these offsets is in element sized units, so we need to convert
523 // to bytes of the individual reads.
524
525 unsigned EltSize;
526 if (LdSt.mayLoad())
527 EltSize = RI.getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
528 else {
529 assert(LdSt.mayStore());
530 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
531 EltSize = RI.getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
532 }
533
534 if (isStride64(Opc))
535 EltSize *= 64;
536
537 BaseOps.push_back(BaseOp);
538 Offset = EltSize * Offset0;
539 // Get appropriate operand(s), and compute width accordingly.
540 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
541 if (DataOpIdx == -1) {
542 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
543 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
544 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
545 Width = LocationSize::precise(
546 Width.getValue() + TypeSize::getFixed(getOpSize(LdSt, DataOpIdx)));
547 } else {
548 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
549 }
550 }
551 return true;
552 }
553
554 if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
555 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
556 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
557 return false;
558 BaseOps.push_back(RSrc);
559 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
560 if (BaseOp && !BaseOp->isFI())
561 BaseOps.push_back(BaseOp);
562 const MachineOperand *OffsetImm =
563 getNamedOperand(LdSt, AMDGPU::OpName::offset);
564 Offset = OffsetImm->getImm();
565 const MachineOperand *SOffset =
566 getNamedOperand(LdSt, AMDGPU::OpName::soffset);
567 if (SOffset) {
568 if (SOffset->isReg())
569 BaseOps.push_back(SOffset);
570 else
571 Offset += SOffset->getImm();
572 }
573 // Get appropriate operand, and compute width accordingly.
574 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
575 if (DataOpIdx == -1)
576 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
577 if (DataOpIdx == -1) // LDS DMA
578 return false;
579 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
580 return true;
581 }
582
583 if (isImage(LdSt)) {
584 auto RsrcOpName =
585 isMIMG(LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
586 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, RsrcOpName);
587 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx));
588 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
589 if (VAddr0Idx >= 0) {
590 // GFX10 possible NSA encoding.
591 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
592 BaseOps.push_back(&LdSt.getOperand(I));
593 } else {
594 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
595 }
596 Offset = 0;
597 // Get appropriate operand, and compute width accordingly.
598 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
599 if (DataOpIdx == -1)
600 return false; // no return sampler
601 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
602 return true;
603 }
604
605 if (isSMRD(LdSt)) {
606 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
607 if (!BaseOp) // e.g. S_MEMTIME
608 return false;
609 BaseOps.push_back(BaseOp);
610 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
611 Offset = OffsetOp ? OffsetOp->getImm() : 0;
612 // Get appropriate operand, and compute width accordingly.
613 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
614 if (DataOpIdx == -1)
615 return false;
616 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
617 return true;
618 }
619
620 if (isFLAT(LdSt)) {
621 // Instructions have either vaddr or saddr or both or none.
622 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
623 if (BaseOp)
624 BaseOps.push_back(BaseOp);
625 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
626 if (BaseOp)
627 BaseOps.push_back(BaseOp);
628 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
629 // Get appropriate operand, and compute width accordingly.
630 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
631 if (DataOpIdx == -1)
632 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
633 if (DataOpIdx == -1) // LDS DMA
634 return false;
635 Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
636 return true;
637 }
638
639 if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&LdSt)) {
640 // The movrel form's index is whatever M0 holds, which can change between
641 // accesses, so it has no base operand to report.
642 if (!LdStIdx->isGPRIdx())
643 return false;
644 BaseOp = &LdStIdx->getIdxOp();
645 OffsetOp = &LdStIdx->getOffsetOp();
646 if (!BaseOp->getReg().isVirtual())
647 return false;
648
649 BaseOps.push_back(BaseOp);
650 Offset = OffsetOp->getImm() * 4; // Offset has units of dwords.
651 Width = LocationSize::precise(LdStIdx->getBitWidth() / 8);
652 return true;
653 }
654
655 return false;
656}
657
658static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
660 const MachineInstr &MI2,
662 // Only examine the first "base" operand of each instruction, on the
663 // assumption that it represents the real base address of the memory access.
664 // Other operands are typically offsets or indices from this base address.
665 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
666 return true;
667
668 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
669 return false;
670
671 auto *MO1 = *MI1.memoperands_begin();
672 auto *MO2 = *MI2.memoperands_begin();
673 if (MO1->getAddrSpace() != MO2->getAddrSpace())
674 return false;
675
676 const auto *Base1 = MO1->getValue();
677 const auto *Base2 = MO2->getValue();
678 if (!Base1 || !Base2)
679 return false;
680 Base1 = getUnderlyingObject(Base1);
681 Base2 = getUnderlyingObject(Base2);
682
683 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
684 return false;
685
686 return Base1 == Base2;
687}
688
691 ArrayRef<const MachineOperand *> BaseOps2) const {
692 // Accesses without base operands (e.g. scratch addressed only by an
693 // immediate offset) share the same implicit base.
694 if (BaseOps1.empty() || BaseOps2.empty())
695 return BaseOps1.empty() && BaseOps2.empty();
696 return memOpsHaveSameBasePtr(*BaseOps1.front()->getParent(), BaseOps1,
697 *BaseOps2.front()->getParent(), BaseOps2);
698}
699
701 int64_t Offset1, bool OffsetIsScalable1,
703 int64_t Offset2, bool OffsetIsScalable2,
704 unsigned ClusterSize,
705 unsigned NumBytes) const {
706 // If the mem ops (to be clustered) do not have the same base ptr, then they
707 // should not be clustered
708 if (!memOpsHaveSameBase(BaseOps1, BaseOps2))
709 return false;
710
711 // Both lists are empty or neither is, so BaseOps1 alone decides whether
712 // there is an instruction to read the budget from.
713 assert(BaseOps1.empty() == BaseOps2.empty());
714 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
715 if (!BaseOps1.empty()) {
716 const MachineFunction *MF = BaseOps1.front()->getParent()->getMF();
717 MaxMemoryClusterDWords =
718 MF->getInfo<SIMachineFunctionInfo>()->getMaxMemoryClusterDWords();
719 }
720
721 // In order to avoid register pressure, on an average, the number of DWORDS
722 // loaded together by all clustered mem ops should not exceed
723 // MaxMemoryClusterDWords. This is an empirical value based on certain
724 // observations and performance related experiments.
725 // The good thing about this heuristic is - it avoids clustering of too many
726 // sub-word loads, and also avoids clustering of wide loads. Below is the
727 // brief summary of how the heuristic behaves for various `LoadSize` when
728 // MaxMemoryClusterDWords is 8.
729 //
730 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
731 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
732 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
733 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
734 // (5) LoadSize >= 17: do not cluster
735 const unsigned LoadSize = NumBytes / ClusterSize;
736 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;
737 return NumDWords <= MaxMemoryClusterDWords;
738}
739
740// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
741// the first 16 loads will be interleaved with the stores, and the next 16 will
742// be clustered as expected. It should really split into 2 16 store batches.
743//
744// Loads are clustered until this returns false, rather than trying to schedule
745// groups of stores. This also means we have to deal with saying different
746// address space loads should be clustered, and ones which might cause bank
747// conflicts.
748//
749// This might be deprecated so it might not be worth that much effort to fix.
751 int64_t Offset0, int64_t Offset1,
752 unsigned NumLoads) const {
753 assert(Offset1 > Offset0 &&
754 "Second offset should be larger than first offset!");
755 // If we have less than 16 loads in a row, and the offsets are within 64
756 // bytes, then schedule together.
757
758 // A cacheline is 64 bytes (for global memory).
759 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
760}
761
764 const DebugLoc &DL, MCRegister DestReg,
765 MCRegister SrcReg, bool KillSrc,
766 const char *Msg = "illegal VGPR to SGPR copy") {
767 MachineFunction *MF = MBB.getParent();
768
771
772 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
773 .addReg(SrcReg, getKillRegState(KillSrc));
774}
775
776/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
777/// possible to have a direct copy in these cases on GFX908, so an intermediate
778/// VGPR copy is required.
781 const DebugLoc &DL, MCRegister DestReg,
782 MCRegister SrcReg, bool KillSrc,
783 RegScavenger &RS, bool RegsOverlap,
784 Register ImpUseSuperReg = Register()) {
785 assert((TII.getSubtarget().hasMAIInsts() &&
786 !TII.getSubtarget().hasGFX90AInsts()) &&
787 "Expected GFX908 subtarget.");
788
789 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
790 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
791 "Source register of the copy should be either an SGPR or an AGPR.");
792
793 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
794 "Destination register of the copy should be an AGPR.");
795
796 const SIRegisterInfo &RI = TII.getRegisterInfo();
797
798 // First try to find defining accvgpr_write to avoid temporary registers.
799 // In the case of copies of overlapping AGPRs, we conservatively do not
800 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
801 // an accvgpr_write used for this same copy due to implicit-defs
802 if (!RegsOverlap) {
803 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
804 --Def;
805
806 if (!Def->modifiesRegister(SrcReg, &RI))
807 continue;
808
809 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
810 Def->getOperand(0).getReg() != SrcReg)
811 break;
812
813 MachineOperand &DefOp = Def->getOperand(1);
814 assert(DefOp.isReg() || DefOp.isImm());
815
816 if (DefOp.isReg()) {
817 bool SafeToPropagate = true;
818 // Check that register source operand is not clobbered before MI.
819 // Immediate operands are always safe to propagate.
820 for (auto I = Def; I != MI && SafeToPropagate; ++I)
821 if (I->modifiesRegister(DefOp.getReg(), &RI))
822 SafeToPropagate = false;
823
824 if (!SafeToPropagate)
825 break;
826
827 for (auto I = Def; I != MI; ++I)
828 I->clearRegisterKills(DefOp.getReg(), &RI);
829 }
830
831 MachineInstrBuilder Builder =
832 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64),
833 DestReg)
834 .add(DefOp);
835
836 if (ImpUseSuperReg) {
837 Builder.addReg(ImpUseSuperReg,
839 }
840
841 return;
842 }
843 }
844
845 RS.enterBasicBlockEnd(MBB);
846 RS.backward(std::next(MI));
847
848 // Ideally we want to have three registers for a long reg_sequence copy
849 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
850 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
851 *MBB.getParent());
852
853 // Registers in the sequence are allocated contiguously so we can just
854 // use register number to pick one of three round-robin temps.
855 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
856 Register Tmp =
857 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
858 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
859 "VGPR used for an intermediate copy should have been reserved.");
860
861 // Only loop through if there are any free registers left. We don't want to
862 // spill.
863 while (RegNo--) {
864 Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI,
865 /* RestoreAfter */ false, 0,
866 /* AllowSpill */ false);
867 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
868 break;
869 Tmp = Tmp2;
870 RS.setRegUsed(Tmp);
871 }
872
873 // Insert copy to temporary VGPR.
874 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
875 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {
876 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
877 } else {
878 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
879 }
880
881 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp)
882 .addReg(SrcReg, getKillRegState(KillSrc));
883 if (ImpUseSuperReg) {
884 UseBuilder.addReg(ImpUseSuperReg,
886 }
887
888 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
889 .addReg(Tmp, RegState::Kill);
890}
891
894 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
895 const TargetRegisterClass *RC, bool Forward) {
896 const SIRegisterInfo &RI = TII.getRegisterInfo();
897 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4);
899 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
900
901 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
902 int16_t SubIdx = BaseIndices[Idx];
903 Register DestSubReg = RI.getSubReg(DestReg, SubIdx);
904 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
905 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
906 unsigned Opcode = AMDGPU::S_MOV_B32;
907
908 // Is SGPR aligned? If so try to combine with next.
909 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
910 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
911 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
912 // Can use SGPR64 copy
913 unsigned Channel = RI.getChannelFromSubReg(SubIdx);
914 SubIdx = RI.getSubRegFromChannel(Channel, 2);
915 DestSubReg = RI.getSubReg(DestReg, SubIdx);
916 SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
917 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
918 Opcode = AMDGPU::S_MOV_B64;
919 Idx++;
920 }
921
922 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), DestSubReg)
923 .addReg(SrcSubReg)
924 .addReg(SrcReg, RegState::Implicit);
925
926 if (!FirstMI)
927 FirstMI = LastMI;
928
929 if (!Forward)
930 I--;
931 }
932
933 assert(FirstMI && LastMI);
934 if (!Forward)
935 std::swap(FirstMI, LastMI);
936
937 if (KillSrc)
938 LastMI->addRegisterKilled(SrcReg, &RI);
939}
940
943 const DebugLoc &DL, Register DestReg,
944 Register SrcReg, bool KillSrc, bool RenamableDest,
945 bool RenamableSrc) const {
946 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DestReg);
947 unsigned Size = RI.getRegSizeInBits(*RC);
948 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg);
949 unsigned SrcSize = RI.getRegSizeInBits(*SrcRC);
950
951 // The rest of copyPhysReg assumes Src and Dst size are the same size.
952 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
953 // we remove fix_16_bit_physreg_copies and this code block?
954 if (ST.getCLOpts().fix_16_bit_physreg_copies) {
955 if (((Size == 16) != (SrcSize == 16))) {
956 // Non-VGPR Src and Dst will later be expanded back to 32 bits.
957 assert(ST.useRealTrue16Insts());
958 Register &RegToFix = (Size == 32) ? DestReg : SrcReg;
959 MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16);
960 RegToFix = SubReg;
961
962 if (DestReg == SrcReg) {
963 // Identity copy. Insert empty bundle since ExpandPostRA expects an
964 // instruction here.
965 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));
966 return;
967 }
968 RC = RI.getPhysRegBaseClass(DestReg);
969 Size = RI.getRegSizeInBits(*RC);
970 SrcRC = RI.getPhysRegBaseClass(SrcReg);
971 SrcSize = RI.getRegSizeInBits(*SrcRC);
972 }
973 }
974
975 if (RC == &AMDGPU::VGPR_32RegClass) {
976 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
977 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
978 AMDGPU::AGPR_32RegClass.contains(SrcReg));
979 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
980 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
981 BuildMI(MBB, MI, DL, get(Opc), DestReg)
982 .addReg(SrcReg, getKillRegState(KillSrc));
983 return;
984 }
985
986 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
987 RC == &AMDGPU::SReg_32RegClass) {
988 if (SrcReg == AMDGPU::SCC) {
989 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
990 .addImm(1)
991 .addImm(0);
992 return;
993 }
994
995 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
996 if (DestReg == AMDGPU::VCC_LO) {
997 // FIXME: Hack until VReg_1 removed.
998 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
999 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
1000 .addImm(0)
1001 .addReg(SrcReg, getKillRegState(KillSrc));
1002 return;
1003 }
1004
1005 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1006 return;
1007 }
1008
1009 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
1010 .addReg(SrcReg, getKillRegState(KillSrc));
1011 return;
1012 }
1013
1014 if (RC == &AMDGPU::SReg_64RegClass) {
1015 if (SrcReg == AMDGPU::SCC) {
1016 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
1017 .addImm(1)
1018 .addImm(0);
1019 return;
1020 }
1021
1022 if (!AMDGPU::SReg_64_EncodableRegClass.contains(SrcReg)) {
1023 if (DestReg == AMDGPU::VCC) {
1024 // FIXME: Hack until VReg_1 removed.
1025 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
1026 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
1027 .addImm(0)
1028 .addReg(SrcReg, getKillRegState(KillSrc));
1029 return;
1030 }
1031
1032 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1033 return;
1034 }
1035
1036 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
1037 .addReg(SrcReg, getKillRegState(KillSrc));
1038 return;
1039 }
1040
1041 if (DestReg == AMDGPU::SCC) {
1042 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
1043 // but SelectionDAG emits such copies for i1 sources.
1044 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
1045 // This copy can only be produced by patterns
1046 // with explicit SCC, which are known to be enabled
1047 // only for subtargets with S_CMP_LG_U64 present.
1048 assert(ST.hasScalarCompareEq64());
1049 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64))
1050 .addReg(SrcReg, getKillRegState(KillSrc))
1051 .addImm(0);
1052 } else {
1053 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
1054 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
1055 .addReg(SrcReg, getKillRegState(KillSrc))
1056 .addImm(0);
1057 }
1058
1059 return;
1060 }
1061
1062 if (RC == &AMDGPU::AGPR_32RegClass) {
1063 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
1064 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) {
1065 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
1066 .addReg(SrcReg, getKillRegState(KillSrc));
1067 return;
1068 }
1069
1070 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) {
1071 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
1072 .addReg(SrcReg, getKillRegState(KillSrc));
1073 return;
1074 }
1075
1076 // FIXME: Pass should maintain scavenger to avoid scan through the block on
1077 // every AGPR spill.
1078 RegScavenger RS;
1079 const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
1080 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, Overlap);
1081 return;
1082 }
1083
1084 if (Size == 16) {
1085 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
1086 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
1087 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
1088
1089 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
1090 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
1091 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
1092 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
1093 bool DstLow = !AMDGPU::isHi16Reg(DestReg, RI);
1094 bool SrcLow = !AMDGPU::isHi16Reg(SrcReg, RI);
1095 MCRegister NewDestReg = RI.get32BitRegister(DestReg);
1096 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg);
1097
1098 if (IsSGPRDst) {
1099 if (!IsSGPRSrc) {
1100 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1101 return;
1102 }
1103
1104 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg)
1105 .addReg(NewSrcReg, getKillRegState(KillSrc));
1106 return;
1107 }
1108
1109 if (IsAGPRDst || IsAGPRSrc) {
1110 if (!DstLow || !SrcLow) {
1111 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1112 "Cannot use hi16 subreg with an AGPR!");
1113 }
1114
1115 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc);
1116 return;
1117 }
1118
1119 if (ST.useRealTrue16Insts()) {
1120 if (IsSGPRSrc) {
1121 assert(SrcLow);
1122 SrcReg = NewSrcReg;
1123 }
1124 // Use the smaller instruction encoding if possible.
1125 if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) &&
1126 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) {
1127 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg)
1128 .addReg(SrcReg);
1129 } else {
1130 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg)
1131 .addImm(0) // src0_modifiers
1132 .addReg(SrcReg)
1133 .addImm(0); // op_sel
1134 }
1135 return;
1136 }
1137
1138 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1139 if (!DstLow || !SrcLow) {
1140 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1141 "Cannot use hi16 subreg on VI!");
1142 }
1143
1144 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)
1145 .addReg(NewSrcReg, getKillRegState(KillSrc));
1146 return;
1147 }
1148
1149 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)
1150 .addImm(0) // src0_modifiers
1151 .addReg(NewSrcReg)
1152 .addImm(0) // clamp
1159 // First implicit operand is $exec.
1160 MIB->tieOperands(0, MIB->getNumOperands() - 1);
1161 return;
1162 }
1163
1164 // Returns true if Dst and Src are in Opc's HwMode-resolved destination and
1165 // source operand classes.
1166 auto CanCopyWith = [&](unsigned Opc, MCRegister Dst, MCRegister Src,
1167 unsigned SrcOp = 1) {
1168 const MCInstrDesc &Desc = get(Opc);
1169 const TargetRegisterClass *DstOpRC = getRegClass(Desc, 0);
1170 const TargetRegisterClass *SrcOpRC = getRegClass(Desc, SrcOp);
1171 return DstOpRC && SrcOpRC && DstOpRC->contains(Dst) &&
1172 SrcOpRC->contains(Src);
1173 };
1174
1175 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {
1176 if (ST.hasVMovB64Inst() &&
1177 CanCopyWith(AMDGPU::V_MOV_B64_e32, DestReg, SrcReg)) {
1178 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg)
1179 .addReg(SrcReg, getKillRegState(KillSrc));
1180 return;
1181 }
1182 if (ST.hasPkMovB32() &&
1183 CanCopyWith(AMDGPU::V_PK_MOV_B32, DestReg, SrcReg, /*SrcOp=*/2)) {
1184 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)
1186 .addReg(SrcReg)
1188 .addReg(SrcReg)
1189 .addImm(0) // op_sel_lo
1190 .addImm(0) // op_sel_hi
1191 .addImm(0) // neg_lo
1192 .addImm(0) // neg_hi
1193 .addImm(0) // clamp
1194 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit);
1195 return;
1196 }
1197 }
1198
1199 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
1200 if (RI.isSGPRClass(RC)) {
1201 if (!RI.isSGPRClass(SrcRC)) {
1202 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1203 return;
1204 }
1205 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
1206 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, CanKillSuperReg, RC,
1207 Forward);
1208 return;
1209 }
1210
1211 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1212 unsigned WideOpcode = AMDGPU::INSTRUCTION_LIST_END;
1213 if (RI.isAGPRClass(RC)) {
1214 if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC))
1215 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1216 else if (RI.hasVGPRs(SrcRC) ||
1217 (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC)))
1218 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1219 else
1220 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1221 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) {
1222 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1223 } else if (RI.isVGPRClass(RC)) {
1224 if (ST.hasVMovB64Inst())
1225 WideOpcode = AMDGPU::V_MOV_B64_e32;
1226 else if (ST.hasPkMovB32())
1227 WideOpcode = AMDGPU::V_PK_MOV_B32;
1228 }
1229
1230 const TargetRegisterClass *WideDstRC{}, *WideSrcRC{};
1231 if (WideOpcode != AMDGPU::INSTRUCTION_LIST_END) {
1232 const MCInstrDesc &Desc = get(WideOpcode);
1233 unsigned SrcOp = WideOpcode == AMDGPU::V_PK_MOV_B32 ? 2 : 1;
1234 WideDstRC = getRegClass(Desc, 0);
1235 WideSrcRC = getRegClass(Desc, SrcOp);
1236 }
1237
1238 // If there is an overlap, we can't kill the super-register on the last
1239 // instruction, since it will also kill the components made live by this def.
1240 const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
1241 const bool CanKillSuperReg = KillSrc && !Overlap;
1242
1243 // For the cases where we need an intermediate instruction/temporary register
1244 // (destination is an AGPR), we need a scavenger.
1245 //
1246 // FIXME: The pass should maintain this for us so we don't have to re-scan the
1247 // whole block for every handled copy.
1248 std::unique_ptr<RegScavenger> RS;
1249 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1250 RS = std::make_unique<RegScavenger>();
1251
1252 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, 4);
1253
1254 for (unsigned Idx{}; Idx < SubIndices.size();) {
1255 unsigned NumRegs = 1;
1256 unsigned ThisOpcode = Opcode;
1257 unsigned SubIdx =
1258 Forward ? SubIndices[Idx] : SubIndices[SubIndices.size() - Idx - 1];
1259
1260 if (WideDstRC && WideSrcRC && Idx + 1 < SubIndices.size()) {
1261 unsigned Channel = RI.getChannelFromSubReg(SubIdx);
1262 if (!Forward)
1263 --Channel;
1264
1265 unsigned WideSubIdx = RI.getSubRegFromChannel(Channel, 2);
1266 Register WideDst = RI.getSubReg(DestReg, WideSubIdx);
1267 Register WideSrc = RI.getSubReg(SrcReg, WideSubIdx);
1268
1269 if (WideDst && WideSrc && WideDstRC->contains(WideDst) &&
1270 WideSrcRC->contains(WideSrc)) {
1271 SubIdx = WideSubIdx;
1272 NumRegs = 2;
1273 ThisOpcode = WideOpcode;
1274 }
1275 }
1276
1277 Register DestSubReg = RI.getSubReg(DestReg, SubIdx);
1278 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
1279 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1280
1281 Idx += NumRegs;
1282 bool UseKill = CanKillSuperReg && Idx == SubIndices.size();
1283
1284 if (ThisOpcode == AMDGPU::INSTRUCTION_LIST_END) {
1285 Register ImpUseSuper = SrcReg;
1286 indirectCopyToAGPR(*this, MBB, MI, DL, DestSubReg, SrcSubReg, UseKill,
1287 *RS, Overlap, ImpUseSuper);
1288 } else if (ThisOpcode == AMDGPU::V_PK_MOV_B32) {
1289 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestSubReg)
1291 .addReg(SrcSubReg)
1293 .addReg(SrcSubReg)
1294 .addImm(0) // op_sel_lo
1295 .addImm(0) // op_sel_hi
1296 .addImm(0) // neg_lo
1297 .addImm(0) // neg_hi
1298 .addImm(0) // clamp
1299 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
1300 } else {
1301 MachineInstrBuilder Builder =
1302 BuildMI(MBB, MI, DL, get(ThisOpcode), DestSubReg).addReg(SrcSubReg);
1303
1304 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
1305 }
1306 }
1307}
1308
1309int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1310 int32_t NewOpc;
1311
1312 // Try to map original to commuted opcode
1313 NewOpc = AMDGPU::getCommuteRev(Opcode);
1314 if (NewOpc != -1)
1315 // Check if the commuted (REV) opcode exists on the target.
1316 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
1317
1318 // Try to map commuted to original opcode
1319 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1320 if (NewOpc != -1)
1321 // Check if the original (non-REV) opcode exists on the target.
1322 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
1323
1324 return Opcode;
1325}
1326
1328 const Register Reg,
1329 int64_t &ImmVal) const {
1330 switch (MI.getOpcode()) {
1331 case AMDGPU::V_MOV_B32_e32:
1332 case AMDGPU::S_MOV_B32:
1333 case AMDGPU::S_MOVK_I32:
1334 case AMDGPU::S_MOV_B64:
1335 case AMDGPU::V_MOV_B64_e32:
1336 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
1337 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
1338 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
1339 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
1340 case AMDGPU::V_MOV_B64_PSEUDO:
1341 case AMDGPU::V_MOV_B16_t16_e32: {
1342 const MachineOperand &Src0 = MI.getOperand(1);
1343 if (Src0.isImm()) {
1344 ImmVal = Src0.getImm();
1345 return MI.getOperand(0).getReg() == Reg;
1346 }
1347
1348 return false;
1349 }
1350 case AMDGPU::V_MOV_B16_t16_e64: {
1351 const MachineOperand &Src0 = MI.getOperand(2);
1352 if (Src0.isImm() && !MI.getOperand(1).getImm()) {
1353 ImmVal = Src0.getImm();
1354 return MI.getOperand(0).getReg() == Reg;
1355 }
1356
1357 return false;
1358 }
1359 case AMDGPU::S_BREV_B32:
1360 case AMDGPU::V_BFREV_B32_e32:
1361 case AMDGPU::V_BFREV_B32_e64: {
1362 const MachineOperand &Src0 = MI.getOperand(1);
1363 if (Src0.isImm()) {
1364 ImmVal = static_cast<int64_t>(reverseBits<int32_t>(Src0.getImm()));
1365 return MI.getOperand(0).getReg() == Reg;
1366 }
1367
1368 return false;
1369 }
1370 case AMDGPU::S_NOT_B32:
1371 case AMDGPU::V_NOT_B32_e32:
1372 case AMDGPU::V_NOT_B32_e64: {
1373 const MachineOperand &Src0 = MI.getOperand(1);
1374 if (Src0.isImm()) {
1375 ImmVal = static_cast<int64_t>(~static_cast<int32_t>(Src0.getImm()));
1376 return MI.getOperand(0).getReg() == Reg;
1377 }
1378
1379 return false;
1380 }
1381 default:
1382 return false;
1383 }
1384}
1385
1386std::optional<int64_t>
1388 const MachineOperand &Op,
1389 MachineInstr **DefMI) const {
1390 if (DefMI)
1391 *DefMI = nullptr;
1392
1393 if (Op.isImm())
1394 return Op.getImm();
1395
1396 if (!Op.isReg() || !Op.getReg().isVirtual())
1397 return std::nullopt;
1398 MachineInstr *Def = MRI.getUniqueVRegDef(Op.getReg());
1399 if (Def && Def->isMoveImmediate()) {
1400 const MachineOperand &ImmSrc = Def->getOperand(1);
1401 if (ImmSrc.isImm()) {
1402 if (DefMI)
1403 *DefMI = Def;
1404 return extractSubregFromImm(ImmSrc.getImm(), Op.getSubReg());
1405 }
1406 }
1407
1408 return std::nullopt;
1409}
1410
1411std::optional<int64_t>
1417
1419
1420 if (RI.isAGPRClass(DstRC))
1421 return AMDGPU::COPY;
1422 if (RI.getRegSizeInBits(*DstRC) == 16) {
1423 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1424 // before RA.
1425 return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1426 }
1427 if (RI.getRegSizeInBits(*DstRC) == 32)
1428 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1429 if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC))
1430 return AMDGPU::S_MOV_B64;
1431 if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC))
1432 return AMDGPU::V_MOV_B64_PSEUDO;
1433 return AMDGPU::COPY;
1434}
1435
1436const MCInstrDesc &
1438 bool IsIndirectSrc) const {
1439 if (IsIndirectSrc) {
1440 if (VecSize <= 32) // 4 bytes
1441 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1442 if (VecSize <= 64) // 8 bytes
1443 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1444 if (VecSize <= 96) // 12 bytes
1445 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1446 if (VecSize <= 128) // 16 bytes
1447 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1448 if (VecSize <= 160) // 20 bytes
1449 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1450 if (VecSize <= 192) // 24 bytes
1451 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6);
1452 if (VecSize <= 224) // 28 bytes
1453 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7);
1454 if (VecSize <= 256) // 32 bytes
1455 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1456 if (VecSize <= 288) // 36 bytes
1457 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1458 if (VecSize <= 320) // 40 bytes
1459 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1460 if (VecSize <= 352) // 44 bytes
1461 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1462 if (VecSize <= 384) // 48 bytes
1463 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1464 if (VecSize <= 512) // 64 bytes
1465 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1466 if (VecSize <= 1024) // 128 bytes
1467 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1468
1469 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1470 }
1471
1472 if (VecSize <= 32) // 4 bytes
1473 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1474 if (VecSize <= 64) // 8 bytes
1475 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1476 if (VecSize <= 96) // 12 bytes
1477 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1478 if (VecSize <= 128) // 16 bytes
1479 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1480 if (VecSize <= 160) // 20 bytes
1481 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1482 if (VecSize <= 192) // 24 bytes
1483 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6);
1484 if (VecSize <= 224) // 28 bytes
1485 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7);
1486 if (VecSize <= 256) // 32 bytes
1487 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1488 if (VecSize <= 288) // 36 bytes
1489 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1490 if (VecSize <= 320) // 40 bytes
1491 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1492 if (VecSize <= 352) // 44 bytes
1493 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1494 if (VecSize <= 384) // 48 bytes
1495 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1496 if (VecSize <= 512) // 64 bytes
1497 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1498 if (VecSize <= 1024) // 128 bytes
1499 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1500
1501 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1502}
1503
1504static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1505 if (VecSize <= 32) // 4 bytes
1506 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1507 if (VecSize <= 64) // 8 bytes
1508 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1509 if (VecSize <= 96) // 12 bytes
1510 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1511 if (VecSize <= 128) // 16 bytes
1512 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1513 if (VecSize <= 160) // 20 bytes
1514 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1515 if (VecSize <= 192) // 24 bytes
1516 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1517 if (VecSize <= 224) // 28 bytes
1518 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1519 if (VecSize <= 256) // 32 bytes
1520 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1521 if (VecSize <= 288) // 36 bytes
1522 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1523 if (VecSize <= 320) // 40 bytes
1524 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1525 if (VecSize <= 352) // 44 bytes
1526 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1527 if (VecSize <= 384) // 48 bytes
1528 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1529 if (VecSize <= 512) // 64 bytes
1530 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1531 if (VecSize <= 1024) // 128 bytes
1532 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1533
1534 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1535}
1536
1537static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1538 if (VecSize <= 32) // 4 bytes
1539 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1540 if (VecSize <= 64) // 8 bytes
1541 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1542 if (VecSize <= 96) // 12 bytes
1543 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1544 if (VecSize <= 128) // 16 bytes
1545 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1546 if (VecSize <= 160) // 20 bytes
1547 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1548 if (VecSize <= 192) // 24 bytes
1549 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1550 if (VecSize <= 224) // 28 bytes
1551 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1552 if (VecSize <= 256) // 32 bytes
1553 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1554 if (VecSize <= 288) // 36 bytes
1555 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1556 if (VecSize <= 320) // 40 bytes
1557 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1558 if (VecSize <= 352) // 44 bytes
1559 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1560 if (VecSize <= 384) // 48 bytes
1561 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1562 if (VecSize <= 512) // 64 bytes
1563 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1564 if (VecSize <= 1024) // 128 bytes
1565 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1566
1567 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1568}
1569
1570static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1571 if (VecSize <= 64) // 8 bytes
1572 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1573 if (VecSize <= 128) // 16 bytes
1574 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1575 if (VecSize <= 256) // 32 bytes
1576 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1577 if (VecSize <= 512) // 64 bytes
1578 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1579 if (VecSize <= 1024) // 128 bytes
1580 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1581
1582 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1583}
1584
1585const MCInstrDesc &
1586SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1587 bool IsSGPR) const {
1588 if (IsSGPR) {
1589 switch (EltSize) {
1590 case 32:
1591 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize));
1592 case 64:
1593 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize));
1594 default:
1595 llvm_unreachable("invalid reg indexing elt size");
1596 }
1597 }
1598
1599 assert(EltSize == 32 && "invalid reg indexing elt size");
1601}
1602
1603static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1604 switch (Size) {
1605 case 4:
1606 return NeedsCFI ? AMDGPU::SI_SPILL_S32_CFI_SAVE : AMDGPU::SI_SPILL_S32_SAVE;
1607 case 8:
1608 return NeedsCFI ? AMDGPU::SI_SPILL_S64_CFI_SAVE : AMDGPU::SI_SPILL_S64_SAVE;
1609 case 12:
1610 return NeedsCFI ? AMDGPU::SI_SPILL_S96_CFI_SAVE : AMDGPU::SI_SPILL_S96_SAVE;
1611 case 16:
1612 return NeedsCFI ? AMDGPU::SI_SPILL_S128_CFI_SAVE
1613 : AMDGPU::SI_SPILL_S128_SAVE;
1614 case 20:
1615 return NeedsCFI ? AMDGPU::SI_SPILL_S160_CFI_SAVE
1616 : AMDGPU::SI_SPILL_S160_SAVE;
1617 case 24:
1618 return NeedsCFI ? AMDGPU::SI_SPILL_S192_CFI_SAVE
1619 : AMDGPU::SI_SPILL_S192_SAVE;
1620 case 28:
1621 return NeedsCFI ? AMDGPU::SI_SPILL_S224_CFI_SAVE
1622 : AMDGPU::SI_SPILL_S224_SAVE;
1623 case 32:
1624 return AMDGPU::SI_SPILL_S256_SAVE;
1625 case 36:
1626 return AMDGPU::SI_SPILL_S288_SAVE;
1627 case 40:
1628 return AMDGPU::SI_SPILL_S320_SAVE;
1629 case 44:
1630 return AMDGPU::SI_SPILL_S352_SAVE;
1631 case 48:
1632 return AMDGPU::SI_SPILL_S384_SAVE;
1633 case 64:
1634 return NeedsCFI ? AMDGPU::SI_SPILL_S512_CFI_SAVE
1635 : AMDGPU::SI_SPILL_S512_SAVE;
1636 case 128:
1637 return NeedsCFI ? AMDGPU::SI_SPILL_S1024_CFI_SAVE
1638 : AMDGPU::SI_SPILL_S1024_SAVE;
1639 default:
1640 llvm_unreachable("unknown register size");
1641 }
1642}
1643
1644static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1645 switch (Size) {
1646 case 2:
1647 return AMDGPU::SI_SPILL_V16_SAVE;
1648 case 4:
1649 return NeedsCFI ? AMDGPU::SI_SPILL_V32_CFI_SAVE : AMDGPU::SI_SPILL_V32_SAVE;
1650 case 8:
1651 return NeedsCFI ? AMDGPU::SI_SPILL_V64_CFI_SAVE : AMDGPU::SI_SPILL_V64_SAVE;
1652 case 12:
1653 return NeedsCFI ? AMDGPU::SI_SPILL_V96_CFI_SAVE : AMDGPU::SI_SPILL_V96_SAVE;
1654 case 16:
1655 return NeedsCFI ? AMDGPU::SI_SPILL_V128_CFI_SAVE
1656 : AMDGPU::SI_SPILL_V128_SAVE;
1657 case 20:
1658 return NeedsCFI ? AMDGPU::SI_SPILL_V160_CFI_SAVE
1659 : AMDGPU::SI_SPILL_V160_SAVE;
1660 case 24:
1661 return NeedsCFI ? AMDGPU::SI_SPILL_V192_CFI_SAVE
1662 : AMDGPU::SI_SPILL_V192_SAVE;
1663 case 28:
1664 return NeedsCFI ? AMDGPU::SI_SPILL_V224_CFI_SAVE
1665 : AMDGPU::SI_SPILL_V224_SAVE;
1666 case 32:
1667 return NeedsCFI ? AMDGPU::SI_SPILL_V256_CFI_SAVE
1668 : AMDGPU::SI_SPILL_V256_SAVE;
1669 case 36:
1670 return NeedsCFI ? AMDGPU::SI_SPILL_V288_CFI_SAVE
1671 : AMDGPU::SI_SPILL_V288_SAVE;
1672 case 40:
1673 return NeedsCFI ? AMDGPU::SI_SPILL_V320_CFI_SAVE
1674 : AMDGPU::SI_SPILL_V320_SAVE;
1675 case 44:
1676 return NeedsCFI ? AMDGPU::SI_SPILL_V352_CFI_SAVE
1677 : AMDGPU::SI_SPILL_V352_SAVE;
1678 case 48:
1679 return NeedsCFI ? AMDGPU::SI_SPILL_V384_CFI_SAVE
1680 : AMDGPU::SI_SPILL_V384_SAVE;
1681 case 64:
1682 return NeedsCFI ? AMDGPU::SI_SPILL_V512_CFI_SAVE
1683 : AMDGPU::SI_SPILL_V512_SAVE;
1684 case 128:
1685 return NeedsCFI ? AMDGPU::SI_SPILL_V1024_CFI_SAVE
1686 : AMDGPU::SI_SPILL_V1024_SAVE;
1687 default:
1688 llvm_unreachable("unknown register size");
1689 }
1690}
1691
1692static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1693 switch (Size) {
1694 case 4:
1695 return NeedsCFI ? AMDGPU::SI_SPILL_AV32_CFI_SAVE
1696 : AMDGPU::SI_SPILL_AV32_SAVE;
1697 case 8:
1698 return NeedsCFI ? AMDGPU::SI_SPILL_AV64_CFI_SAVE
1699 : AMDGPU::SI_SPILL_AV64_SAVE;
1700 case 12:
1701 return NeedsCFI ? AMDGPU::SI_SPILL_AV96_CFI_SAVE
1702 : AMDGPU::SI_SPILL_AV96_SAVE;
1703 case 16:
1704 return NeedsCFI ? AMDGPU::SI_SPILL_AV128_CFI_SAVE
1705 : AMDGPU::SI_SPILL_AV128_SAVE;
1706 case 20:
1707 return NeedsCFI ? AMDGPU::SI_SPILL_AV160_CFI_SAVE
1708 : AMDGPU::SI_SPILL_AV160_SAVE;
1709 case 24:
1710 return NeedsCFI ? AMDGPU::SI_SPILL_AV192_CFI_SAVE
1711 : AMDGPU::SI_SPILL_AV192_SAVE;
1712 case 28:
1713 return NeedsCFI ? AMDGPU::SI_SPILL_AV224_CFI_SAVE
1714 : AMDGPU::SI_SPILL_AV224_SAVE;
1715 case 32:
1716 return NeedsCFI ? AMDGPU::SI_SPILL_AV256_CFI_SAVE
1717 : AMDGPU::SI_SPILL_AV256_SAVE;
1718 case 36:
1719 return AMDGPU::SI_SPILL_AV288_SAVE;
1720 case 40:
1721 return AMDGPU::SI_SPILL_AV320_SAVE;
1722 case 44:
1723 return AMDGPU::SI_SPILL_AV352_SAVE;
1724 case 48:
1725 return AMDGPU::SI_SPILL_AV384_SAVE;
1726 case 64:
1727 return NeedsCFI ? AMDGPU::SI_SPILL_AV512_CFI_SAVE
1728 : AMDGPU::SI_SPILL_AV512_SAVE;
1729 case 128:
1730 return NeedsCFI ? AMDGPU::SI_SPILL_AV1024_CFI_SAVE
1731 : AMDGPU::SI_SPILL_AV1024_SAVE;
1732 default:
1733 llvm_unreachable("unknown register size");
1734 }
1735}
1736
1737static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1738 bool IsVectorSuperClass) {
1739 // Currently, there is only 32-bit WWM register spills needed.
1740 if (Size != 4)
1741 llvm_unreachable("unknown wwm register spill size");
1742
1743 if (IsVectorSuperClass)
1744 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1745
1746 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1747}
1748
1750 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1751 const SIMachineFunctionInfo &MFI, bool NeedsCFI) const {
1752 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1753
1754 // Choose the right opcode if spilling a WWM register.
1756 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1757
1758 // TODO: Check if AGPRs are available
1759 if (ST.hasMAIInsts())
1760 return getAVSpillSaveOpcode(Size, NeedsCFI);
1761
1762 return getVGPRSpillSaveOpcode(Size, NeedsCFI);
1763}
1764
1765void SIInstrInfo::storeRegToStackSlotImpl(
1767 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1768 MachineInstr::MIFlag Flags, bool NeedsCFI) const {
1769 MachineFunction *MF = MBB.getParent();
1771 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1772 const DebugLoc &DL = MBB.findDebugLoc(MI);
1773
1774 MachinePointerInfo PtrInfo
1775 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1777 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex),
1778 FrameInfo.getObjectAlign(FrameIndex));
1779 unsigned SpillSize = RI.getSpillSize(*RC);
1780
1782 if (RI.isSGPRClass(RC)) {
1783 if (FrameInfo.getStackID(FrameIndex) == TargetStackID::SGPRSpill)
1784 MFI->setHasSpilledSGPRs();
1785 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1786 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1787 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1788
1789 // We are only allowed to create one new instruction when spilling
1790 // registers, so we need to use pseudo instruction for spilling SGPRs.
1791 const MCInstrDesc &OpDesc =
1792 get(getSGPRSpillSaveOpcode(SpillSize, NeedsCFI));
1793
1794 // The SGPR spill/restore instructions only work on number sgprs, so we need
1795 // to make sure we are using the correct register class.
1796 if (SrcReg.isVirtual() && SpillSize == 4) {
1797 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1798 }
1799
1800 BuildMI(MBB, MI, DL, OpDesc)
1801 .addReg(SrcReg, getKillRegState(isKill)) // data
1802 .addFrameIndex(FrameIndex) // addr
1803 .addMemOperand(MMO)
1805
1806 return;
1807 }
1808
1809 unsigned Opcode = getVectorRegSpillSaveOpcode(VReg ? VReg : SrcReg, RC,
1810 SpillSize, *MFI, NeedsCFI);
1811 MFI->setHasSpilledVGPRs();
1812
1813 BuildMI(MBB, MI, DL, get(Opcode))
1814 .addReg(SrcReg, getKillRegState(isKill)) // data
1815 .addFrameIndex(FrameIndex) // addr
1816 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1817 .addImm(0) // offset
1818 .addMemOperand(MMO);
1819}
1820
1823 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1824 MachineInstr::MIFlag Flags) const {
1825 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
1826 false);
1827}
1828
1831 Register SrcReg, bool isKill,
1832 int FrameIndex,
1833 const TargetRegisterClass *RC) const {
1834 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, Register(),
1835 MachineInstr::NoFlags, true);
1836}
1837
1838static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1839 switch (Size) {
1840 case 4:
1841 return AMDGPU::SI_SPILL_S32_RESTORE;
1842 case 8:
1843 return AMDGPU::SI_SPILL_S64_RESTORE;
1844 case 12:
1845 return AMDGPU::SI_SPILL_S96_RESTORE;
1846 case 16:
1847 return AMDGPU::SI_SPILL_S128_RESTORE;
1848 case 20:
1849 return AMDGPU::SI_SPILL_S160_RESTORE;
1850 case 24:
1851 return AMDGPU::SI_SPILL_S192_RESTORE;
1852 case 28:
1853 return AMDGPU::SI_SPILL_S224_RESTORE;
1854 case 32:
1855 return AMDGPU::SI_SPILL_S256_RESTORE;
1856 case 36:
1857 return AMDGPU::SI_SPILL_S288_RESTORE;
1858 case 40:
1859 return AMDGPU::SI_SPILL_S320_RESTORE;
1860 case 44:
1861 return AMDGPU::SI_SPILL_S352_RESTORE;
1862 case 48:
1863 return AMDGPU::SI_SPILL_S384_RESTORE;
1864 case 64:
1865 return AMDGPU::SI_SPILL_S512_RESTORE;
1866 case 128:
1867 return AMDGPU::SI_SPILL_S1024_RESTORE;
1868 default:
1869 llvm_unreachable("unknown register size");
1870 }
1871}
1872
1873static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1874 switch (Size) {
1875 case 2:
1876 return AMDGPU::SI_SPILL_V16_RESTORE;
1877 case 4:
1878 return AMDGPU::SI_SPILL_V32_RESTORE;
1879 case 8:
1880 return AMDGPU::SI_SPILL_V64_RESTORE;
1881 case 12:
1882 return AMDGPU::SI_SPILL_V96_RESTORE;
1883 case 16:
1884 return AMDGPU::SI_SPILL_V128_RESTORE;
1885 case 20:
1886 return AMDGPU::SI_SPILL_V160_RESTORE;
1887 case 24:
1888 return AMDGPU::SI_SPILL_V192_RESTORE;
1889 case 28:
1890 return AMDGPU::SI_SPILL_V224_RESTORE;
1891 case 32:
1892 return AMDGPU::SI_SPILL_V256_RESTORE;
1893 case 36:
1894 return AMDGPU::SI_SPILL_V288_RESTORE;
1895 case 40:
1896 return AMDGPU::SI_SPILL_V320_RESTORE;
1897 case 44:
1898 return AMDGPU::SI_SPILL_V352_RESTORE;
1899 case 48:
1900 return AMDGPU::SI_SPILL_V384_RESTORE;
1901 case 64:
1902 return AMDGPU::SI_SPILL_V512_RESTORE;
1903 case 128:
1904 return AMDGPU::SI_SPILL_V1024_RESTORE;
1905 default:
1906 llvm_unreachable("unknown register size");
1907 }
1908}
1909
1910static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1911 switch (Size) {
1912 case 4:
1913 return AMDGPU::SI_SPILL_AV32_RESTORE;
1914 case 8:
1915 return AMDGPU::SI_SPILL_AV64_RESTORE;
1916 case 12:
1917 return AMDGPU::SI_SPILL_AV96_RESTORE;
1918 case 16:
1919 return AMDGPU::SI_SPILL_AV128_RESTORE;
1920 case 20:
1921 return AMDGPU::SI_SPILL_AV160_RESTORE;
1922 case 24:
1923 return AMDGPU::SI_SPILL_AV192_RESTORE;
1924 case 28:
1925 return AMDGPU::SI_SPILL_AV224_RESTORE;
1926 case 32:
1927 return AMDGPU::SI_SPILL_AV256_RESTORE;
1928 case 36:
1929 return AMDGPU::SI_SPILL_AV288_RESTORE;
1930 case 40:
1931 return AMDGPU::SI_SPILL_AV320_RESTORE;
1932 case 44:
1933 return AMDGPU::SI_SPILL_AV352_RESTORE;
1934 case 48:
1935 return AMDGPU::SI_SPILL_AV384_RESTORE;
1936 case 64:
1937 return AMDGPU::SI_SPILL_AV512_RESTORE;
1938 case 128:
1939 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1940 default:
1941 llvm_unreachable("unknown register size");
1942 }
1943}
1944
1945static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1946 bool IsVectorSuperClass) {
1947 // Currently, there is only 32-bit WWM register spills needed.
1948 if (Size != 4)
1949 llvm_unreachable("unknown wwm register spill size");
1950
1951 if (IsVectorSuperClass) // TODO: Always use this if there are AGPRs
1952 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1953
1954 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1955}
1956
1958 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1959 const SIMachineFunctionInfo &MFI) const {
1960 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1961
1962 // Choose the right opcode if restoring a WWM register.
1964 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1965
1966 // TODO: Check if AGPRs are available
1967 if (ST.hasMAIInsts())
1969
1970 assert(!RI.isAGPRClass(RC));
1972}
1973
1976 Register DestReg, int FrameIndex,
1977 const TargetRegisterClass *RC,
1978 Register VReg, unsigned SubReg,
1979 MachineInstr::MIFlag Flags) const {
1980 MachineFunction *MF = MBB.getParent();
1982 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1983 const DebugLoc &DL = MBB.findDebugLoc(MI);
1984 unsigned SpillSize = RI.getSpillSize(*RC);
1985
1986 MachinePointerInfo PtrInfo
1987 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1988
1990 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
1991 FrameInfo.getObjectAlign(FrameIndex));
1992
1993 if (RI.isSGPRClass(RC)) {
1994 if (FrameInfo.getStackID(FrameIndex) == TargetStackID::SGPRSpill)
1995 MFI->setHasSpilledSGPRs();
1996 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1997 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1998 DestReg != AMDGPU::EXEC && "exec should not be spilled");
1999
2000 // FIXME: Maybe this should not include a memoperand because it will be
2001 // lowered to non-memory instructions.
2002 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
2003 if (DestReg.isVirtual() && SpillSize == 4) {
2005 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
2006 }
2007
2008 BuildMI(MBB, MI, DL, OpDesc, DestReg)
2009 .addFrameIndex(FrameIndex) // addr
2010 .addMemOperand(MMO)
2012
2013 return;
2014 }
2015
2016 unsigned Opcode = getVectorRegSpillRestoreOpcode(VReg ? VReg : DestReg, RC,
2017 SpillSize, *MFI);
2018 BuildMI(MBB, MI, DL, get(Opcode), DestReg)
2019 .addFrameIndex(FrameIndex) // vaddr
2020 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
2021 .addImm(0) // offset
2022 .addMemOperand(MMO);
2023}
2024
2029
2032 unsigned Quantity) const {
2033 DebugLoc DL = MBB.findDebugLoc(MI);
2034 unsigned MaxSNopCount = 1u << ST.getSNopBits();
2035 while (Quantity > 0) {
2036 unsigned Arg = std::min(Quantity, MaxSNopCount);
2037 Quantity -= Arg;
2038 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1);
2039 }
2040}
2041
2045 const DebugLoc &DL) const {
2046 MachineFunction *MF = MBB.getParent();
2047 constexpr unsigned DoorbellIDMask = 0x3ff;
2048 constexpr unsigned ECQueueWaveAbort = 0x400;
2049
2050 MachineBasicBlock *TrapBB = &MBB;
2051 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();
2052
2053 if (!MBB.succ_empty() ||
2055 /*SkipPseudoOp=*/false) != MBB.end()) {
2056 MBB.splitAt(MI, /*UpdateLiveIns=*/false);
2057 TrapBB = MF->CreateMachineBasicBlock();
2058 BuildMI(MBB, MI, DL, get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(TrapBB);
2059 MF->push_back(TrapBB);
2060 MBB.addSuccessor(TrapBB);
2061 }
2062 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this
2063 // will be a nop.
2064 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_TRAP))
2065 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
2066 Register DoorbellReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2067 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_SENDMSG_RTN_B32),
2068 DoorbellReg)
2070 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::TTMP2)
2071 .addUse(AMDGPU::M0);
2072 Register DoorbellRegMasked =
2073 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2074 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_AND_B32), DoorbellRegMasked)
2075 .addUse(DoorbellReg)
2076 .addImm(DoorbellIDMask)
2077 .setOperandDead(3); // implicit-def $scc
2078 Register SetWaveAbortBit =
2079 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2080 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_OR_B32), SetWaveAbortBit)
2081 .addUse(DoorbellRegMasked)
2082 .addImm(ECQueueWaveAbort)
2083 .setOperandDead(3); // implicit-def $scc
2084 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::M0)
2085 .addUse(SetWaveAbortBit);
2086 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_SENDMSG))
2088 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::M0)
2089 .addUse(AMDGPU::TTMP2);
2090 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_BRANCH)).addMBB(HaltLoopBB);
2091 TrapBB->addSuccessor(HaltLoopBB);
2092
2093 BuildMI(*HaltLoopBB, HaltLoopBB->end(), DL, get(AMDGPU::S_SETHALT)).addImm(5);
2094 BuildMI(*HaltLoopBB, HaltLoopBB->end(), DL, get(AMDGPU::S_BRANCH))
2095 .addMBB(HaltLoopBB);
2096 MF->push_back(HaltLoopBB);
2097 HaltLoopBB->addSuccessor(HaltLoopBB);
2098
2099 return MBB.getNextNode();
2100}
2101
2103 switch (MI.getOpcode()) {
2104 default:
2105 if (MI.isMetaInstruction())
2106 return 0;
2107 return 1; // FIXME: Do wait states equal cycles?
2108
2109 case AMDGPU::S_NOP:
2110 return MI.getOperand(0).getImm() + 1;
2111 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
2112 // hazard, even if one exist, won't really be visible. Should we handle it?
2113 }
2114}
2115
2116// Expand a VGPR "as memory" access into one indexed move per dword.
2119 MachineBasicBlock &MBB = *MI.getParent();
2120 MachineFunction &MF = *MBB.getParent();
2121 const DebugLoc &DL = MI.getDebugLoc();
2122 const SIRegisterInfo &RI = TII.getRegisterInfo();
2123 const bool IsStore = LdSt.mayStore();
2124
2125 Register Data = LdSt.getDataOp().getReg();
2126 unsigned Offset = LdSt.getOffsetOp().getImm();
2127 unsigned NumDwords = LdSt.getBitWidth() / 32;
2128
2129 const bool UseGPRIdxMode = LdSt.isGPRIdx();
2130 assert(UseGPRIdxMode == TII.getSubtarget().useVGPRIndexMode());
2131
2132 MachineInstr *SetOn = nullptr;
2133 if (UseGPRIdxMode) {
2134 SetOn = BuildMI(MBB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_ON))
2135 .add(LdSt.getIdxOp())
2138 }
2139
2140 unsigned Opcode;
2141 if (UseGPRIdxMode) {
2142 Opcode = IsStore ? AMDGPU::V_MOV_B32_indirect_write
2143 : AMDGPU::V_MOV_B32_indirect_read;
2144 } else {
2145 Opcode =
2146 IsStore ? AMDGPU::V_MOVRELD_B32_as_mem : AMDGPU::V_MOVRELS_B32_as_mem;
2147 }
2148
2149 // A move touches its base plus the run-time index, which no operand can name,
2150 // so the base is undef and the liveness of those registers is not expressed.
2151 const RegState DataFlags = IsStore
2152 ? getUndefRegState(LdSt.getDataOp().isUndef())
2154 for (unsigned i = 0; i < NumDwords; ++i) {
2155 Register Base = AMDGPU::VGPR_32RegClass.getRegister(Offset + i);
2156 Register Sub = Data;
2157 if (NumDwords != 1)
2158 Sub = RI.getSubReg(Data, RI.getSubRegFromChannel(i));
2159
2161 if (IsStore) {
2162 Move = BuildMI(MBB, MI, DL, TII.get(Opcode))
2164 .addReg(Sub, DataFlags);
2165 // As in copyPhysReg, the implicit use of the whole tuple keeps a partly
2166 // undefined value a valid operand.
2167 if (NumDwords != 1)
2168 Move.addReg(Data, RegState::Implicit | DataFlags);
2169 } else {
2170 Move = BuildMI(MBB, MI, DL, TII.get(Opcode), Sub)
2172 }
2173 // Each mode switch defines M0, which keeps the index-mode moves in order. A
2174 // movrel move only reads M0, so it is a load or store, ordered by the
2175 // memory operand of the dword it moves.
2176 if (!UseGPRIdxMode) {
2177 for (const MachineMemOperand *MMO : MI.memoperands())
2178 Move.addMemOperand(MF.getMachineMemOperand(MMO, 4 * i, 4));
2179 }
2180 }
2181
2182 // Bundled so the post-RA scheduler keeps everything else out of the mode
2183 // switch.
2184 if (SetOn) {
2185 MachineInstr *SetOff =
2186 BuildMI(MBB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_OFF));
2187 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
2188 }
2189
2190 MI.eraseFromParent();
2191}
2192
2195 expandLoadStoreIdx(*this, MI);
2196 return true;
2197 }
2198
2199 MachineBasicBlock &MBB = *MI.getParent();
2200 DebugLoc DL = MBB.findDebugLoc(MI);
2202
2203 switch (MI.getOpcode()) {
2204 default: return TargetInstrInfo::expandPostRAPseudo(MI);
2205 case AMDGPU::S_MOV_B64_term:
2206 // This is only a terminator to get the correct spill code placement during
2207 // register allocation.
2208 MI.setDesc(get(AMDGPU::S_MOV_B64));
2209 break;
2210
2211 case AMDGPU::S_MOV_B32_term:
2212 // This is only a terminator to get the correct spill code placement during
2213 // register allocation.
2214 MI.setDesc(get(AMDGPU::S_MOV_B32));
2215 break;
2216
2217 case AMDGPU::S_XOR_B64_term:
2218 // This is only a terminator to get the correct spill code placement during
2219 // register allocation.
2220 MI.setDesc(get(AMDGPU::S_XOR_B64));
2221 break;
2222
2223 case AMDGPU::S_XOR_B32_term:
2224 // This is only a terminator to get the correct spill code placement during
2225 // register allocation.
2226 MI.setDesc(get(AMDGPU::S_XOR_B32));
2227 break;
2228 case AMDGPU::S_OR_B64_term:
2229 // This is only a terminator to get the correct spill code placement during
2230 // register allocation.
2231 MI.setDesc(get(AMDGPU::S_OR_B64));
2232 break;
2233 case AMDGPU::S_OR_B32_term:
2234 // This is only a terminator to get the correct spill code placement during
2235 // register allocation.
2236 MI.setDesc(get(AMDGPU::S_OR_B32));
2237 break;
2238
2239 case AMDGPU::S_ANDN2_B64_term:
2240 // This is only a terminator to get the correct spill code placement during
2241 // register allocation.
2242 MI.setDesc(get(AMDGPU::S_ANDN2_B64));
2243 break;
2244
2245 case AMDGPU::S_ANDN2_B32_term:
2246 // This is only a terminator to get the correct spill code placement during
2247 // register allocation.
2248 MI.setDesc(get(AMDGPU::S_ANDN2_B32));
2249 break;
2250
2251 case AMDGPU::S_AND_B64_term:
2252 // This is only a terminator to get the correct spill code placement during
2253 // register allocation.
2254 MI.setDesc(get(AMDGPU::S_AND_B64));
2255 break;
2256
2257 case AMDGPU::S_AND_B32_term:
2258 // This is only a terminator to get the correct spill code placement during
2259 // register allocation.
2260 MI.setDesc(get(AMDGPU::S_AND_B32));
2261 break;
2262
2263 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2264 // This is only a terminator to get the correct spill code placement during
2265 // register allocation.
2266 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B64));
2267 break;
2268
2269 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2270 // This is only a terminator to get the correct spill code placement during
2271 // register allocation.
2272 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32));
2273 break;
2274
2275 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
2276 MI.setDesc(get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
2277 break;
2278 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
2279 MI.setDesc(get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
2280 break;
2281
2282 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2283 MI.setDesc(get(AMDGPU::V_WRITELANE_B32));
2284 break;
2285
2286 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2287 MI.setDesc(get(AMDGPU::V_READLANE_B32));
2288 break;
2289 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
2290 Register Dst = MI.getOperand(0).getReg();
2291 bool IsAGPR = SIRegisterInfo::isAGPRClass(RI.getPhysRegBaseClass(Dst));
2292 MI.setDesc(
2293 get(IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
2294 break;
2295 }
2296 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
2297 Register Dst = MI.getOperand(0).getReg();
2298 if (SIRegisterInfo::isAGPRClass(RI.getPhysRegBaseClass(Dst))) {
2299 int64_t Imm = MI.getOperand(1).getImm();
2300
2301 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2302 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2303 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DstLo)
2305 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DstHi)
2306 .addImm(SignExtend64<32>(Imm >> 32));
2307 MI.eraseFromParent();
2308 break;
2309 }
2310
2311 [[fallthrough]];
2312 }
2313 case AMDGPU::V_MOV_B64_PSEUDO: {
2314 Register Dst = MI.getOperand(0).getReg();
2315 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2316 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2317
2318 const MCInstrDesc &Mov64Desc = get(AMDGPU::V_MOV_B64_e32);
2319 const TargetRegisterClass *Mov64RC = getRegClass(Mov64Desc, /*OpNum=*/0);
2320
2321 const MachineOperand &SrcOp = MI.getOperand(1);
2322 // FIXME: Will this work for 64-bit floating point immediates?
2323 assert(!SrcOp.isFPImm());
2324 if (ST.hasVMovB64Inst() && Mov64RC->contains(Dst)) {
2325 MI.setDesc(Mov64Desc);
2326 if (SrcOp.isReg() || isInlineConstant(MI, 1) ||
2327 (SrcOp.isImm() &&
2328 (isUInt<32>(SrcOp.getImm()) || ST.has64BitLiterals())) ||
2329 (SrcOp.isGlobal() && ST.has64BitLiterals()))
2330 break;
2331 }
2332 if (SrcOp.isGlobal()) {
2333 // The address is unknown until link time, so the PK_MOV inline-constant
2334 // shortcut cannot apply.
2335 const GlobalValue *GV = SrcOp.getGlobal();
2336 int64_t Offset = SrcOp.getOffset();
2337 auto [LoFlags, HiFlags] = splitGlobalAddressRelocFlags(SrcOp);
2338
2339 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
2340 .addGlobalAddress(GV, Offset, LoFlags);
2341 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
2342 .addGlobalAddress(GV, Offset, HiFlags);
2343 } else if (SrcOp.isImm()) {
2344 APInt Imm(64, SrcOp.getImm());
2345 APInt Lo(32, Imm.getLoBits(32).getZExtValue());
2346 APInt Hi(32, Imm.getHiBits(32).getZExtValue());
2347 const MCInstrDesc &PkMovDesc = get(AMDGPU::V_PK_MOV_B32);
2348 const TargetRegisterClass *PkMovRC = getRegClass(PkMovDesc, /*OpNum=*/0);
2349
2350 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Lo) &&
2351 PkMovRC->contains(Dst)) {
2352 BuildMI(MBB, MI, DL, PkMovDesc, Dst)
2354 .addImm(Lo.getSExtValue())
2356 .addImm(Lo.getSExtValue())
2357 .addImm(0) // op_sel_lo
2358 .addImm(0) // op_sel_hi
2359 .addImm(0) // neg_lo
2360 .addImm(0) // neg_hi
2361 .addImm(0); // clamp
2362 } else {
2363 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
2364 .addImm(Lo.getSExtValue());
2365 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
2366 .addImm(Hi.getSExtValue());
2367 }
2368 } else {
2369 assert(SrcOp.isReg());
2370 if (ST.hasPkMovB32() &&
2371 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) {
2372 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
2373 .addImm(SISrcMods::OP_SEL_1) // src0_mod
2374 .addReg(SrcOp.getReg())
2376 .addReg(SrcOp.getReg())
2377 .addImm(0) // op_sel_lo
2378 .addImm(0) // op_sel_hi
2379 .addImm(0) // neg_lo
2380 .addImm(0) // neg_hi
2381 .addImm(0); // clamp
2382 } else {
2383 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
2384 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0));
2385 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
2386 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1));
2387 }
2388 }
2389 MI.eraseFromParent();
2390 break;
2391 }
2392 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2394 break;
2395 }
2396 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2397 const MachineOperand &SrcOp = MI.getOperand(1);
2398 assert(!SrcOp.isFPImm());
2399
2400 if (ST.has64BitLiterals()) {
2401 MI.setDesc(get(AMDGPU::S_MOV_B64));
2402 break;
2403 }
2404
2405 if (SrcOp.isGlobal()) {
2406 Register Dst = MI.getOperand(0).getReg();
2407 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2408 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2409 const GlobalValue *GV = SrcOp.getGlobal();
2410 int64_t Offset = SrcOp.getOffset();
2411 auto [LoFlags, HiFlags] = splitGlobalAddressRelocFlags(SrcOp);
2412
2413 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo)
2414 .addGlobalAddress(GV, Offset, LoFlags);
2415 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi)
2416 .addGlobalAddress(GV, Offset, HiFlags);
2417 MI.eraseFromParent();
2418 break;
2419 }
2420
2421 // SrcOp is immediate
2422 APInt Imm(64, SrcOp.getImm());
2423 if (Imm.isIntN(32) || isInlineConstant(Imm)) {
2424 MI.setDesc(get(AMDGPU::S_MOV_B64));
2425 break;
2426 }
2427
2428 Register Dst = MI.getOperand(0).getReg();
2429 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2430 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2431
2432 APInt Lo(32, Imm.getLoBits(32).getZExtValue());
2433 APInt Hi(32, Imm.getHiBits(32).getZExtValue());
2434 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo)
2435 .addImm(Lo.getSExtValue());
2436 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi)
2437 .addImm(Hi.getSExtValue());
2438 MI.eraseFromParent();
2439 break;
2440 }
2441 case AMDGPU::V_SET_INACTIVE_B32: {
2442 // Lower V_SET_INACTIVE_B32 to V_CNDMASK_B32.
2443 Register DstReg = MI.getOperand(0).getReg();
2444 BuildMI(MBB, MI, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
2445 .add(MI.getOperand(3))
2446 .add(MI.getOperand(4))
2447 .add(MI.getOperand(1))
2448 .add(MI.getOperand(2))
2449 .add(MI.getOperand(5));
2450 MI.eraseFromParent();
2451 break;
2452 }
2453 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2454 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2455 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2456 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2457 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2458 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2459 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2460 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2461 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2462 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2463 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2464 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2465 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2466 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2467 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2468 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2469 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2470 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2471 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2472 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2473 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2474 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2475 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2476 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2477 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2478 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2479 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2480 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2481 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2482 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2483 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2484 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2485 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2486 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2);
2487
2488 unsigned Opc;
2489 if (RI.hasVGPRs(EltRC)) {
2490 Opc = AMDGPU::V_MOVRELD_B32_e32;
2491 } else {
2492 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2493 : AMDGPU::S_MOVRELD_B32;
2494 }
2495
2496 const MCInstrDesc &OpDesc = get(Opc);
2497 Register VecReg = MI.getOperand(0).getReg();
2498 bool IsUndef = MI.getOperand(1).isUndef();
2499 unsigned SubReg = MI.getOperand(3).getImm();
2500 assert(VecReg == MI.getOperand(1).getReg());
2501
2503 BuildMI(MBB, MI, DL, OpDesc)
2504 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2505 .add(MI.getOperand(2))
2507 .addReg(VecReg, RegState::Implicit | getUndefRegState(IsUndef));
2508
2509 const int ImpDefIdx =
2510 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2511 const int ImpUseIdx = ImpDefIdx + 1;
2512 MIB->tieOperands(ImpDefIdx, ImpUseIdx);
2513 MI.eraseFromParent();
2514 break;
2515 }
2516 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2517 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2518 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2519 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2520 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2521 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6:
2522 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7:
2523 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2524 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2525 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2526 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2527 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2528 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2529 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2530 assert(ST.useVGPRIndexMode());
2531 Register VecReg = MI.getOperand(0).getReg();
2532 bool IsUndef = MI.getOperand(1).isUndef();
2533 MachineOperand &Idx = MI.getOperand(3);
2534 Register SubReg = MI.getOperand(4).getImm();
2535
2536 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2537 .add(Idx)
2539 SetOn->getOperand(3).setIsUndef();
2540
2541 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write);
2543 BuildMI(MBB, MI, DL, OpDesc)
2544 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2545 .add(MI.getOperand(2))
2547 .addReg(VecReg, RegState::Implicit | getUndefRegState(IsUndef));
2548
2549 const int ImpDefIdx =
2550 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2551 const int ImpUseIdx = ImpDefIdx + 1;
2552 MIB->tieOperands(ImpDefIdx, ImpUseIdx);
2553
2554 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2555
2556 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
2557
2558 MI.eraseFromParent();
2559 break;
2560 }
2561 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2562 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2563 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2564 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2565 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2566 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6:
2567 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7:
2568 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2569 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2570 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2571 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2572 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2573 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2574 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2575 assert(ST.useVGPRIndexMode());
2576 Register Dst = MI.getOperand(0).getReg();
2577 Register VecReg = MI.getOperand(1).getReg();
2578 bool IsUndef = MI.getOperand(1).isUndef();
2579 Register SubReg = MI.getOperand(3).getImm();
2580
2581 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2582 .add(MI.getOperand(2))
2584 SetOn->getOperand(3).setIsUndef();
2585
2586 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read))
2587 .addDef(Dst)
2588 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2589 .addReg(VecReg, RegState::Implicit | getUndefRegState(IsUndef));
2590
2591 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2592
2593 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
2594
2595 MI.eraseFromParent();
2596 break;
2597 }
2598 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2599 MachineFunction &MF = *MBB.getParent();
2600 Register Reg = MI.getOperand(0).getReg();
2601 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
2602 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
2603 MachineOperand OpLo = MI.getOperand(1);
2604 MachineOperand OpHi = MI.getOperand(2);
2605
2606 // Create a bundle so these instructions won't be re-ordered by the
2607 // post-RA scheduler.
2608 MIBundleBuilder Bundler(MBB, MI);
2609 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
2610
2611 // What we want here is an offset from the value returned by s_getpc (which
2612 // is the address of the s_add_u32 instruction) to the global variable, but
2613 // since the encoding of $symbol starts 4 bytes after the start of the
2614 // s_add_u32 instruction, we end up with an offset that is 4 bytes too
2615 // small. This requires us to add 4 to the global variable offset in order
2616 // to compute the correct address. Similarly for the s_addc_u32 instruction,
2617 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2618 // instruction.
2619
2620 int64_t Adjust = 0;
2621 if (ST.hasGetPCZeroExtension()) {
2622 // Fix up hardware that does not sign-extend the 48-bit PC value by
2623 // inserting: s_sext_i32_i16 reghi, reghi
2624 Bundler.append(
2625 BuildMI(MF, DL, get(AMDGPU::S_SEXT_I32_I16), RegHi).addReg(RegHi));
2626 Adjust += 4;
2627 }
2628
2629 if (OpLo.isGlobal())
2630 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2631 Bundler.append(
2632 BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo));
2633
2634 if (OpHi.isGlobal())
2635 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2636 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
2637 .addReg(RegHi)
2638 .add(OpHi));
2639
2640 finalizeBundle(MBB, Bundler.begin());
2641
2642 MI.eraseFromParent();
2643 break;
2644 }
2645 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {
2646 MachineFunction &MF = *MBB.getParent();
2647 Register Reg = MI.getOperand(0).getReg();
2648 MachineOperand Op = MI.getOperand(1);
2649
2650 // Create a bundle so these instructions won't be re-ordered by the
2651 // post-RA scheduler.
2652 MIBundleBuilder Bundler(MBB, MI);
2653 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
2654 if (Op.isGlobal())
2655 Op.setOffset(Op.getOffset() + 4);
2656 Bundler.append(
2657 BuildMI(MF, DL, get(AMDGPU::S_ADD_U64), Reg).addReg(Reg).add(Op));
2658
2659 finalizeBundle(MBB, Bundler.begin());
2660
2661 MI.eraseFromParent();
2662 break;
2663 }
2664 case AMDGPU::ENTER_STRICT_WWM: {
2665 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2666 // Whole Wave Mode is entered.
2667 MI.setDesc(get(LMC.OrSaveExecOpc));
2668 break;
2669 }
2670 case AMDGPU::ENTER_STRICT_WQM: {
2671 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2672 // STRICT_WQM is entered.
2673 BuildMI(MBB, MI, DL, get(LMC.MovOpc), MI.getOperand(0).getReg())
2674 .addReg(LMC.ExecReg);
2675 BuildMI(MBB, MI, DL, get(LMC.WQMOpc), LMC.ExecReg).addReg(LMC.ExecReg);
2676
2677 MI.eraseFromParent();
2678 break;
2679 }
2680 case AMDGPU::EXIT_STRICT_WWM:
2681 case AMDGPU::EXIT_STRICT_WQM: {
2682 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2683 // WWM/STICT_WQM is exited.
2684 MI.setDesc(get(LMC.MovOpc));
2685 break;
2686 }
2687 case AMDGPU::SI_RETURN: {
2688 const MachineFunction *MF = MBB.getParent();
2689 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2690 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2691 // Hiding the return address use with SI_RETURN may lead to extra kills in
2692 // the function and missing live-ins. We are fine in practice because callee
2693 // saved register handling ensures the register value is restored before
2694 // RET, but we need the undef flag here to appease the MachineVerifier
2695 // liveness checks.
2697 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return))
2698 .addReg(TRI->getReturnAddressReg(*MF), RegState::Undef);
2699
2700 MIB.copyImplicitOps(MI);
2701 MI.eraseFromParent();
2702 break;
2703 }
2704
2705 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2706 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2707 MI.setDesc(get(AMDGPU::S_MUL_U64));
2708 break;
2709
2710 case AMDGPU::S_GETPC_B64_pseudo:
2711 MI.setDesc(get(AMDGPU::S_GETPC_B64));
2712 if (ST.hasGetPCZeroExtension()) {
2713 Register Dst = MI.getOperand(0).getReg();
2714 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2715 // Fix up hardware that does not sign-extend the 48-bit PC value by
2716 // inserting: s_sext_i32_i16 dsthi, dsthi
2717 BuildMI(MBB, std::next(MI.getIterator()), DL, get(AMDGPU::S_SEXT_I32_I16),
2718 DstHi)
2719 .addReg(DstHi);
2720 }
2721 break;
2722
2723 case AMDGPU::V_MAX_BF16_PSEUDO_e64: {
2724 assert(ST.hasBF16PackedInsts());
2725 MI.setDesc(get(AMDGPU::V_PK_MAX_NUM_BF16));
2726 MI.addOperand(MachineOperand::CreateImm(0)); // op_sel
2727 MI.addOperand(MachineOperand::CreateImm(0)); // neg_lo
2728 MI.addOperand(MachineOperand::CreateImm(0)); // neg_hi
2729 auto Op0 = getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2730 Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
2731 auto Op1 = getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2732 Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
2733 break;
2734 }
2735
2736 case AMDGPU::GET_STACK_BASE:
2737 // The stack starts at offset 0 unless we need to reserve some space at the
2738 // bottom.
2739 if (ST.getFrameLowering()->mayReserveScratchForCWSR(*MBB.getParent())) {
2740 // When CWSR is used in dynamic VGPR mode, the trap handler needs to save
2741 // some of the VGPRs. The size of the required scratch space has already
2742 // been computed by prolog epilog insertion.
2743 const SIMachineFunctionInfo *MFI =
2744 MBB.getParent()->getInfo<SIMachineFunctionInfo>();
2745 unsigned VGPRSize = MFI->getScratchReservedForDynamicVGPRs();
2746 Register DestReg = MI.getOperand(0).getReg();
2747 BuildMI(MBB, MI, DL, get(AMDGPU::S_GETREG_B32), DestReg)
2750 // The MicroEngine ID is 0 for the graphics queue, and 1 or 2 for compute
2751 // (3 is unused, so we ignore it). Unfortunately, S_GETREG doesn't set
2752 // SCC, so we need to check for 0 manually.
2753 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)).addImm(0).addReg(DestReg);
2754 // Change the implicif-def of SCC to an explicit use (but first remove
2755 // the dead flag if present).
2756 MI.getOperand(MI.getNumExplicitOperands()).setIsDead(false);
2757 MI.getOperand(MI.getNumExplicitOperands()).setIsUse();
2758 MI.setDesc(get(AMDGPU::S_CMOVK_I32));
2759 MI.addOperand(MachineOperand::CreateImm(VGPRSize));
2760 } else {
2761 MI.setDesc(get(AMDGPU::S_MOV_B32));
2762 MI.addOperand(MachineOperand::CreateImm(0));
2763 MI.removeOperand(
2764 MI.getNumExplicitOperands()); // Drop implicit def of SCC.
2765 }
2766 break;
2767 }
2768
2769 return true;
2770}
2771
2774 unsigned SubIdx, const MachineInstr &Orig,
2775 LaneBitmask UsedLanes) const {
2776
2777 // Try shrinking the instruction to remat only the part needed for current
2778 // context.
2779 // TODO: Handle more cases.
2780 unsigned Opcode = Orig.getOpcode();
2781 switch (Opcode) {
2782 case AMDGPU::S_MOV_B64:
2783 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2784 if (SubIdx != 0)
2785 break;
2786
2787 if (!Orig.getOperand(1).isImm())
2788 break;
2789
2790 // Shrink S_MOV_B64 to S_MOV_B32 when UsedLanes indicates only a single
2791 // 32-bit lane of the 64-bit value is live at the rematerialization point.
2792 if (UsedLanes.all())
2793 break;
2794
2795 // Determine which half of the 64-bit immediate corresponds to the use.
2796 unsigned OrigSubReg = Orig.getOperand(0).getSubReg();
2797 unsigned LoSubReg = RI.composeSubRegIndices(OrigSubReg, AMDGPU::sub0);
2798 unsigned HiSubReg = RI.composeSubRegIndices(OrigSubReg, AMDGPU::sub1);
2799
2800 bool NeedLo = (UsedLanes & RI.getSubRegIndexLaneMask(LoSubReg)).any();
2801 bool NeedHi = (UsedLanes & RI.getSubRegIndexLaneMask(HiSubReg)).any();
2802
2803 if (NeedLo && NeedHi)
2804 break;
2805
2806 int64_t Imm64 = Orig.getOperand(1).getImm();
2807 int32_t Imm32 = NeedLo ? Lo_32(Imm64) : Hi_32(Imm64);
2808
2809 unsigned UseSubReg = NeedLo ? LoSubReg : HiSubReg;
2810
2811 // Emit S_MOV_B32 defining just the needed 32-bit subreg of DestReg.
2812 BuildMI(MBB, I, Orig.getDebugLoc(), get(AMDGPU::S_MOV_B32))
2813 .addReg(DestReg, RegState::Define | RegState::Undef, UseSubReg)
2814 .addImm(Imm32);
2815 return;
2816 }
2817
2818 case AMDGPU::S_LOAD_DWORDX16_IMM:
2819 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2820 if (SubIdx != 0 || UsedLanes.all())
2821 break;
2822
2823 if (I == MBB.end())
2824 break;
2825
2826 if (I->isBundled())
2827 break;
2828
2829 MachineFunction *MF = MBB.getParent();
2831 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2832
2833 // Look for a single use of the register at the insertion point that is also
2834 // a subreg.
2835 Register RegToFind = Orig.getOperand(0).getReg();
2836 MachineOperand *UseMO = MRI.getOneNonDBGUse(RegToFind);
2837 if (!UseMO ||
2838 UseMO != I->findRegisterUseOperand(RegToFind, /*TRI=*/nullptr) ||
2839 UseMO->getSubReg() == AMDGPU::NoSubRegister)
2840 break;
2841
2842 unsigned Offset = RI.getSubRegIdxOffset(UseMO->getSubReg());
2843 unsigned SubregSize = RI.getSubRegIdxSize(UseMO->getSubReg());
2844
2845 unsigned NewOpcode = -1;
2846 if (SubregSize == 256)
2847 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2848 else if (SubregSize == 128)
2849 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2850 else
2851 break;
2852
2853 const MCInstrDesc &TID = get(NewOpcode);
2854 const TargetRegisterClass *NewRC =
2855 RI.getAllocatableClass(getRegClass(TID, 0));
2856 MRI.setRegClass(DestReg, NewRC);
2857
2858 UseMO->setReg(DestReg);
2859 UseMO->setSubReg(AMDGPU::NoSubRegister);
2860
2861 // Use a smaller load with the desired size, possibly with updated offset.
2862 MachineInstr *MI = MF->CloneMachineInstr(&Orig);
2863 MI->setDesc(TID);
2864 MI->getOperand(0).setReg(DestReg);
2865 MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister);
2866 if (Offset) {
2867 MachineOperand *OffsetMO = getNamedOperand(*MI, AMDGPU::OpName::offset);
2868 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2869 OffsetMO->setImm(FinalOffset);
2870 }
2872 for (const MachineMemOperand *MemOp : Orig.memoperands())
2873 NewMMOs.push_back(MF->getMachineMemOperand(MemOp, MemOp->getPointerInfo(),
2874 SubregSize / 8));
2875 MI->setMemRefs(*MF, NewMMOs);
2876
2877 MBB.insert(I, MI);
2878 return;
2879 }
2880
2881 default:
2882 break;
2883 }
2884
2885 TargetInstrInfo::reMaterialize(MBB, I, DestReg, SubIdx, Orig, UsedLanes);
2886}
2887
2888std::pair<MachineInstr*, MachineInstr*>
2890 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2891
2892 if (ST.hasVMovB64Inst() && ST.hasFeature(AMDGPU::FeatureDPALU_DPP) &&
2894 ST, getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
2895 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));
2896 return std::pair(&MI, nullptr);
2897 }
2898
2899 MachineBasicBlock &MBB = *MI.getParent();
2900 DebugLoc DL = MBB.findDebugLoc(MI);
2901 MachineFunction *MF = MBB.getParent();
2903 Register Dst = MI.getOperand(0).getReg();
2904 unsigned Part = 0;
2905 MachineInstr *Split[2];
2906
2907 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2908 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
2909 if (Dst.isPhysical()) {
2910 MovDPP.addDef(RI.getSubReg(Dst, Sub));
2911 } else {
2912 assert(MRI.isSSA());
2913 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2914 MovDPP.addDef(Tmp);
2915 }
2916
2917 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2918 const MachineOperand &SrcOp = MI.getOperand(I);
2919 assert(!SrcOp.isFPImm());
2920 if (SrcOp.isImm()) {
2921 APInt Imm(64, SrcOp.getImm());
2922 Imm.ashrInPlace(Part * 32);
2923 MovDPP.addImm(Imm.getLoBits(32).getZExtValue());
2924 } else {
2925 assert(SrcOp.isReg());
2926 Register Src = SrcOp.getReg();
2927 if (Src.isPhysical())
2928 MovDPP.addReg(RI.getSubReg(Src, Sub));
2929 else
2930 MovDPP.addReg(Src, getUndefRegState(SrcOp.isUndef()), Sub);
2931 }
2932 }
2933
2934 for (const MachineOperand &MO : llvm::drop_begin(MI.explicit_operands(), 3))
2935 MovDPP.addImm(MO.getImm());
2936
2937 Split[Part] = MovDPP;
2938 ++Part;
2939 }
2940
2941 if (Dst.isVirtual())
2942 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
2943 .addReg(Split[0]->getOperand(0).getReg())
2944 .addImm(AMDGPU::sub0)
2945 .addReg(Split[1]->getOperand(0).getReg())
2946 .addImm(AMDGPU::sub1);
2947
2948 MI.eraseFromParent();
2949 return std::pair(Split[0], Split[1]);
2950}
2951
2952std::optional<DestSourcePair>
2954 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2955 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2956
2957 return std::nullopt;
2958}
2959
2961 AMDGPU::OpName Src0OpName,
2962 MachineOperand &Src1,
2963 AMDGPU::OpName Src1OpName) const {
2964 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
2965 if (!Src0Mods)
2966 return false;
2967
2968 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
2969 assert(Src1Mods &&
2970 "All commutable instructions have both src0 and src1 modifiers");
2971
2972 int Src0ModsVal = Src0Mods->getImm();
2973 int Src1ModsVal = Src1Mods->getImm();
2974
2975 Src1Mods->setImm(Src0ModsVal);
2976 Src0Mods->setImm(Src1ModsVal);
2977 return true;
2978}
2979
2981 MachineOperand &RegOp,
2982 MachineOperand &NonRegOp) {
2983 Register Reg = RegOp.getReg();
2984 unsigned SubReg = RegOp.getSubReg();
2985 bool IsKill = RegOp.isKill();
2986 bool IsDead = RegOp.isDead();
2987 bool IsUndef = RegOp.isUndef();
2988 bool IsDebug = RegOp.isDebug();
2989
2990 if (NonRegOp.isImm())
2991 RegOp.ChangeToImmediate(NonRegOp.getImm());
2992 else if (NonRegOp.isFI())
2993 RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
2994 else if (NonRegOp.isGlobal()) {
2995 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(),
2996 NonRegOp.getTargetFlags());
2997 } else
2998 return nullptr;
2999
3000 // Make sure we don't reinterpret a subreg index in the target flags.
3001 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
3002
3003 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
3004 NonRegOp.setSubReg(SubReg);
3005
3006 return &MI;
3007}
3008
3010 MachineOperand &NonRegOp1,
3011 MachineOperand &NonRegOp2) {
3012 unsigned TargetFlags = NonRegOp1.getTargetFlags();
3013 int64_t NonRegVal = NonRegOp1.getImm();
3014
3015 NonRegOp1.setImm(NonRegOp2.getImm());
3016 NonRegOp2.setImm(NonRegVal);
3017 NonRegOp1.setTargetFlags(NonRegOp2.getTargetFlags());
3018 NonRegOp2.setTargetFlags(TargetFlags);
3019 return &MI;
3020}
3021
3022// GFX950 v_pk_{add,mul}_f32 must not have opsel[1:0] == 2'b10, i.e. OP_SEL_0 on
3023// src1 but not on src0, and v_pk_fma_f32 must not have OP_SEL_0 on src1 or
3024// src2.
3026 bool CheckSwappedSrc = false) {
3027 unsigned Opc = MI.getOpcode();
3028 if (!TII.getSubtarget().hasPkF32InvalidOpSel() ||
3029 (Opc != AMDGPU::V_PK_ADD_F32 && Opc != AMDGPU::V_PK_MUL_F32 &&
3030 Opc != AMDGPU::V_PK_FMA_F32))
3031 return true;
3032
3033 auto Mods = [&](AMDGPU::OpName Name) {
3034 const MachineOperand *Op = TII.getNamedOperand(MI, Name);
3035 return Op ? static_cast<unsigned>(Op->getImm()) : 0u;
3036 };
3037 unsigned Src0Mods = Mods(AMDGPU::OpName::src0_modifiers);
3038 unsigned Src1Mods = Mods(AMDGPU::OpName::src1_modifiers);
3039 unsigned Src2Mods = Mods(AMDGPU::OpName::src2_modifiers);
3040
3041 if (CheckSwappedSrc)
3042 std::swap(Src0Mods, Src1Mods);
3043
3044 if (Opc == AMDGPU::V_PK_FMA_F32)
3045 return !((Src1Mods | Src2Mods) & SISrcMods::OP_SEL_0);
3046 return (Src0Mods & SISrcMods::OP_SEL_0) || !(Src1Mods & SISrcMods::OP_SEL_0);
3047}
3048
3049bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
3050 unsigned OpIdx1) const {
3051 const MCInstrDesc &InstDesc = MI.getDesc();
3052 const MCOperandInfo &OpInfo0 = InstDesc.operands()[OpIdx0];
3053 const MCOperandInfo &OpInfo1 = InstDesc.operands()[OpIdx1];
3054
3055 unsigned Opc = MI.getOpcode();
3056 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3057 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3058
3059 // Swapping src0 and src1 swaps their modifiers; refuse a swap that would
3060 // leave an invalid GFX950 packed FP32 op_sel.
3061 bool IsSrc01Swap = ((int)OpIdx0 == Src0Idx && (int)OpIdx1 == Src1Idx) ||
3062 ((int)OpIdx0 == Src1Idx && (int)OpIdx1 == Src0Idx);
3063 if (IsSrc01Swap && !isLegalPkF32OpSel(*this, MI, true))
3064 return false;
3065
3066 const MachineOperand &MO0 = MI.getOperand(OpIdx0);
3067 const MachineOperand &MO1 = MI.getOperand(OpIdx1);
3068
3069 // Swap doesn't breach constant bus or literal limits, but it may move a
3070 // constant out of src0. Allow that for immediates where the destination
3071 // operand can hold them, e.g. VOP3 sources on targets with VOP3 literal
3072 // support. Other non-register operands stay in src0.
3073 if (isVALU(MI, /*AllowLDSDMA=*/false)) {
3074 if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&
3075 (!MO0.isImm() || !isImmOperandLegal(MI, OpIdx1, MO0)))
3076 return false;
3077 if ((int)OpIdx1 == Src0Idx && !MO1.isReg() &&
3078 (!MO1.isImm() || !isImmOperandLegal(MI, OpIdx0, MO1)))
3079 return false;
3080 }
3081
3082 if ((int)OpIdx1 != Src0Idx && MO0.isReg()) {
3083 if (OpInfo1.RegClass == -1)
3084 return OpInfo1.OperandType == MCOI::OPERAND_UNKNOWN;
3085 return isLegalRegOperand(MI, OpIdx1, MO0) &&
3086 (!MO1.isReg() || isLegalRegOperand(MI, OpIdx0, MO1));
3087 }
3088 if ((int)OpIdx0 != Src0Idx && MO1.isReg()) {
3089 if (OpInfo0.RegClass == -1)
3090 return OpInfo0.OperandType == MCOI::OPERAND_UNKNOWN;
3091 return (!MO0.isReg() || isLegalRegOperand(MI, OpIdx1, MO0)) &&
3092 isLegalRegOperand(MI, OpIdx0, MO1);
3093 }
3094
3095 // No need to check 64-bit literals since swapping does not bring new
3096 // 64-bit literals into current instruction to fold to 32-bit
3097
3098 return isImmOperandLegal(MI, OpIdx1, MO0);
3099}
3100
3102 if (!isDPP(MI))
3103 return false;
3104 const MachineOperand *DppCtrl = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3105 return !DppCtrl || DppCtrl->getImm() != AMDGPU::DPP::QUAD_PERM_ID;
3106}
3107
3109 unsigned Src0Idx,
3110 unsigned Src1Idx) const {
3111 assert(!NewMI && "this should never be used");
3112
3114 return nullptr;
3115
3116 unsigned Opc = MI.getOpcode();
3117 int CommutedOpcode = commuteOpcode(Opc);
3118 if (CommutedOpcode == -1)
3119 return nullptr;
3120
3121 if (Src0Idx > Src1Idx)
3122 std::swap(Src0Idx, Src1Idx);
3123
3124 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
3125 static_cast<int>(Src0Idx) &&
3126 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
3127 static_cast<int>(Src1Idx) &&
3128 "inconsistency with findCommutedOpIndices");
3129
3130 if (!isLegalToSwap(MI, Src0Idx, Src1Idx))
3131 return nullptr;
3132
3133 MachineInstr *CommutedMI = nullptr;
3134 MachineOperand &Src0 = MI.getOperand(Src0Idx);
3135 MachineOperand &Src1 = MI.getOperand(Src1Idx);
3136 if (Src0.isReg() && Src1.isReg()) {
3137 // Be sure to copy the source modifiers to the right place.
3138 CommutedMI =
3139 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
3140 } else if (Src0.isReg() && !Src1.isReg()) {
3141 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
3142 } else if (!Src0.isReg() && Src1.isReg()) {
3143 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
3144 } else if (Src0.isImm() && Src1.isImm()) {
3145 CommutedMI = swapImmOperands(MI, Src0, Src1);
3146 } else {
3147 // FIXME: Found two non registers to commute. This does happen.
3148 return nullptr;
3149 }
3150
3151 if (CommutedMI) {
3152 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
3153 Src1, AMDGPU::OpName::src1_modifiers);
3154
3155 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_sel, Src1,
3156 AMDGPU::OpName::src1_sel);
3157
3158 CommutedMI->setDesc(get(CommutedOpcode));
3159 }
3160
3161 return CommutedMI;
3162}
3163
3164// This needs to be implemented because the source modifiers may be inserted
3165// between the true commutable operands, and the base
3166// TargetInstrInfo::commuteInstruction uses it.
3168 unsigned &SrcOpIdx0,
3169 unsigned &SrcOpIdx1) const {
3171 return false;
3172
3173 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
3174}
3175
3177 unsigned &SrcOpIdx0,
3178 unsigned &SrcOpIdx1) const {
3179 if (!Desc.isCommutable())
3180 return false;
3181
3182 unsigned Opc = Desc.getOpcode();
3183 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3184 if (Src0Idx == -1)
3185 return false;
3186
3187 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3188 if (Src1Idx == -1)
3189 return false;
3190
3191 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
3192}
3193
3195 int64_t BrOffset) const {
3196 // BranchRelaxation should never have to check s_setpc_b64 or s_add_pc_i64
3197 // because its dest block is unanalyzable.
3198 assert(isSOPP(BranchOp) || isSOPK(BranchOp));
3199
3200 // Convert to dwords.
3201 BrOffset /= 4;
3202
3203 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
3204 // from the next instruction.
3205 BrOffset -= 1;
3206
3207 return isIntN(ST.getCLOpts().s_branch_bits, BrOffset);
3208}
3209
3212 return MI.getOperand(0).getMBB();
3213}
3214
3216 for (const MachineInstr &MI : MBB->terminators()) {
3217 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
3218 MI.getOpcode() == AMDGPU::SI_LOOP ||
3219 MI.getOpcode() == AMDGPU::SI_WATERFALL_LOOP)
3220 return true;
3221 }
3222 return false;
3223}
3224
3226 MachineBasicBlock &DestBB,
3227 MachineBasicBlock &RestoreBB,
3228 const DebugLoc &DL, int64_t BrOffset,
3229 RegScavenger *RS) const {
3230 assert(MBB.empty() &&
3231 "new block should be inserted for expanding unconditional branch");
3232 assert(MBB.pred_size() == 1);
3233 assert(RestoreBB.empty() &&
3234 "restore block should be inserted for restoring clobbered registers");
3235
3236 MachineFunction *MF = MBB.getParent();
3239 auto I = MBB.end();
3240 auto &MCCtx = MF->getContext();
3241
3242 if (ST.useAddPCI64Inst()) {
3243 MCSymbol *Offset =
3244 MCCtx.createTempSymbol("offset", /*AlwaysAddSuffix=*/true);
3245 auto AddPC = BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_PC_I64))
3247 MCSymbol *PostAddPCLabel =
3248 MCCtx.createTempSymbol("post_addpc", /*AlwaysAddSuffix=*/true);
3249 AddPC->setPostInstrSymbol(*MF, PostAddPCLabel);
3250 auto *OffsetExpr = MCBinaryExpr::createSub(
3251 MCSymbolRefExpr::create(DestBB.getSymbol(), MCCtx),
3252 MCSymbolRefExpr::create(PostAddPCLabel, MCCtx), MCCtx);
3253 Offset->setVariableValue(OffsetExpr);
3254 return;
3255 }
3256
3257 assert(RS && "RegScavenger required for long branching");
3258
3259 // FIXME: Virtual register workaround for RegScavenger not working with empty
3260 // blocks.
3261 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3262
3263 // Note: as this is used after hazard recognizer we need to apply some hazard
3264 // workarounds directly.
3265 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
3266 ST.hasVALUReadSGPRHazard();
3267 auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {
3268 if (FlushSGPRWrites)
3269 BuildMI(MBB, I, DL, get(AMDGPU::S_WAITCNT_DEPCTR))
3271 };
3272
3273 // We need to compute the offset relative to the instruction immediately after
3274 // s_getpc_b64. Insert pc arithmetic code before last terminator.
3275 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
3276 ApplyHazardWorkarounds();
3277
3278 MCSymbol *PostGetPCLabel =
3279 MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true);
3280 GetPC->setPostInstrSymbol(*MF, PostGetPCLabel);
3281
3282 MCSymbol *OffsetLo =
3283 MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true);
3284 MCSymbol *OffsetHi =
3285 MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true);
3286 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
3287 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
3288 .addReg(PCReg, {}, AMDGPU::sub0)
3289 .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET);
3290 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
3291 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
3292 .addReg(PCReg, {}, AMDGPU::sub1)
3293 .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET);
3294 ApplyHazardWorkarounds();
3295
3296 // Insert the indirect branch after the other terminator.
3297 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
3298 .addReg(PCReg);
3299
3300 // If a spill is needed for the pc register pair, we need to insert a spill
3301 // restore block right before the destination block, and insert a short branch
3302 // into the old destination block's fallthrough predecessor.
3303 // e.g.:
3304 //
3305 // s_cbranch_scc0 skip_long_branch:
3306 //
3307 // long_branch_bb:
3308 // spill s[8:9]
3309 // s_getpc_b64 s[8:9]
3310 // s_add_u32 s8, s8, restore_bb
3311 // s_addc_u32 s9, s9, 0
3312 // s_setpc_b64 s[8:9]
3313 //
3314 // skip_long_branch:
3315 // foo;
3316 //
3317 // .....
3318 //
3319 // dest_bb_fallthrough_predecessor:
3320 // bar;
3321 // s_branch dest_bb
3322 //
3323 // restore_bb:
3324 // restore s[8:9]
3325 // fallthrough dest_bb
3326 ///
3327 // dest_bb:
3328 // buzz;
3329
3330 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
3331 Register Scav;
3332
3333 // If we've previously reserved a register for long branches
3334 // avoid running the scavenger and just use those registers
3335 if (LongBranchReservedReg) {
3336 RS->enterBasicBlock(MBB);
3337 Scav = LongBranchReservedReg;
3338 } else {
3339 RS->enterBasicBlockEnd(MBB);
3340 Scav = RS->scavengeRegisterBackwards(
3341 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC),
3342 /* RestoreAfter */ false, 0, /* AllowSpill */ false);
3343 }
3344 if (Scav) {
3345 RS->setRegUsed(Scav);
3346 MRI.replaceRegWith(PCReg, Scav);
3347 MRI.clearVirtRegs();
3348 } else {
3349 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
3350 // SGPR spill.
3351 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3352 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3353 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS);
3354 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1);
3355 MRI.clearVirtRegs();
3356 }
3357
3358 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
3359 // Now, the distance could be defined.
3361 MCSymbolRefExpr::create(DestLabel, MCCtx),
3362 MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx);
3363 // Add offset assignments.
3364 auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx);
3365 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx));
3366 auto *ShAmt = MCConstantExpr::create(32, MCCtx);
3367 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx));
3368}
3369
3370unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
3371 switch (Cond) {
3372 case SIInstrInfo::SCC_TRUE:
3373 return AMDGPU::S_CBRANCH_SCC1;
3374 case SIInstrInfo::SCC_FALSE:
3375 return AMDGPU::S_CBRANCH_SCC0;
3376 case SIInstrInfo::VCCNZ:
3377 return AMDGPU::S_CBRANCH_VCCNZ;
3378 case SIInstrInfo::VCCZ:
3379 return AMDGPU::S_CBRANCH_VCCZ;
3380 case SIInstrInfo::EXECNZ:
3381 return AMDGPU::S_CBRANCH_EXECNZ;
3382 case SIInstrInfo::EXECZ:
3383 return AMDGPU::S_CBRANCH_EXECZ;
3384 default:
3385 llvm_unreachable("invalid branch predicate");
3386 }
3387}
3388
3389SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
3390 switch (Opcode) {
3391 case AMDGPU::S_CBRANCH_SCC0:
3392 return SCC_FALSE;
3393 case AMDGPU::S_CBRANCH_SCC1:
3394 return SCC_TRUE;
3395 case AMDGPU::S_CBRANCH_VCCNZ:
3396 return VCCNZ;
3397 case AMDGPU::S_CBRANCH_VCCZ:
3398 return VCCZ;
3399 case AMDGPU::S_CBRANCH_EXECNZ:
3400 return EXECNZ;
3401 case AMDGPU::S_CBRANCH_EXECZ:
3402 return EXECZ;
3403 default:
3404 return INVALID_BR;
3405 }
3406}
3407
3411 MachineBasicBlock *&FBB,
3413 bool AllowModify) const {
3414 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3415 // Unconditional Branch
3416 TBB = I->getOperand(0).getMBB();
3417 return false;
3418 }
3419
3420 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
3421 if (Pred == INVALID_BR)
3422 return true;
3423
3424 MachineBasicBlock *CondBB = I->getOperand(0).getMBB();
3425 Cond.push_back(MachineOperand::CreateImm(Pred));
3426 Cond.push_back(I->getOperand(1)); // Save the branch register.
3427
3428 ++I;
3429
3430 if (I == MBB.end()) {
3431 // Conditional branch followed by fall-through.
3432 TBB = CondBB;
3433 return false;
3434 }
3435
3436 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3437 TBB = CondBB;
3438 FBB = I->getOperand(0).getMBB();
3439 return false;
3440 }
3441
3442 return true;
3443}
3444
3446 MachineBasicBlock *&FBB,
3448 bool AllowModify) const {
3449 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3450 auto E = MBB.end();
3451 if (I == E)
3452 return false;
3453
3454 // Skip over the instructions that are artificially terminators for special
3455 // exec management.
3456 while (I != E && !I->isBranch() && !I->isReturn()) {
3457 switch (I->getOpcode()) {
3458 case AMDGPU::S_MOV_B64_term:
3459 case AMDGPU::S_XOR_B64_term:
3460 case AMDGPU::S_OR_B64_term:
3461 case AMDGPU::S_ANDN2_B64_term:
3462 case AMDGPU::S_AND_B64_term:
3463 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3464 case AMDGPU::S_MOV_B32_term:
3465 case AMDGPU::S_XOR_B32_term:
3466 case AMDGPU::S_OR_B32_term:
3467 case AMDGPU::S_ANDN2_B32_term:
3468 case AMDGPU::S_AND_B32_term:
3469 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3470 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
3471 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
3472 break;
3473 case AMDGPU::SI_IF:
3474 case AMDGPU::SI_ELSE:
3475 case AMDGPU::SI_KILL_I1_TERMINATOR:
3476 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3477 // FIXME: It's messy that these need to be considered here at all.
3478 return true;
3479 default:
3480 llvm_unreachable("unexpected non-branch terminator inst");
3481 }
3482
3483 ++I;
3484 }
3485
3486 if (I == E)
3487 return false;
3488
3489 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3490}
3491
3493 int *BytesRemoved) const {
3494 unsigned Count = 0;
3495 unsigned RemovedSize = 0;
3496 for (MachineInstr &MI : llvm::make_early_inc_range(MBB.terminators())) {
3497 // Skip over artificial terminators when removing instructions.
3498 if (MI.isBranch() || MI.isReturn()) {
3499 RemovedSize += getInstSizeInBytes(MI);
3500 MI.eraseFromParent();
3501 ++Count;
3502 }
3503 }
3504
3505 if (BytesRemoved)
3506 *BytesRemoved = RemovedSize;
3507
3508 return Count;
3509}
3510
3511// Copy the flags onto the implicit condition register operand.
3513 const MachineOperand &OrigCond) {
3514 CondReg.setIsUndef(OrigCond.isUndef());
3515 CondReg.setIsKill(OrigCond.isKill());
3516}
3517
3520 MachineBasicBlock *FBB,
3522 const DebugLoc &DL,
3523 int *BytesAdded) const {
3524 if (!FBB && Cond.empty()) {
3525 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
3526 .addMBB(TBB);
3527 if (BytesAdded)
3528 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3529 return 1;
3530 }
3531
3532 assert(TBB && Cond[0].isImm());
3533
3534 unsigned Opcode
3535 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
3536
3537 if (!FBB) {
3538 MachineInstr *CondBr =
3539 BuildMI(&MBB, DL, get(Opcode))
3540 .addMBB(TBB);
3541
3542 // Copy the flags onto the implicit condition register operand.
3543 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
3544 fixImplicitOperands(*CondBr);
3545
3546 if (BytesAdded)
3547 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3548 return 1;
3549 }
3550
3551 assert(TBB && FBB);
3552
3553 MachineInstr *CondBr =
3554 BuildMI(&MBB, DL, get(Opcode))
3555 .addMBB(TBB);
3556 fixImplicitOperands(*CondBr);
3557 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
3558 .addMBB(FBB);
3559
3560 MachineOperand &CondReg = CondBr->getOperand(1);
3561 CondReg.setIsUndef(Cond[1].isUndef());
3562 CondReg.setIsKill(Cond[1].isKill());
3563
3564 if (BytesAdded)
3565 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3566
3567 return 2;
3568}
3569
3572 if (Cond.size() != 2) {
3573 return true;
3574 }
3575
3576 if (Cond[0].isImm()) {
3577 Cond[0].setImm(-Cond[0].getImm());
3578 return false;
3579 }
3580
3581 return true;
3582}
3583
3584namespace {
3585class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
3586private:
3587 /// The compare instruction for loop control
3588 const MachineInstr *CmpInst = nullptr;
3589 /// The normalized condition used by createTripCountGreaterCondition()
3591
3592public:
3593 AMDGPUPipelinerLoopInfo(MachineInstr *CmpInst,
3595 : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()) {}
3596
3597 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
3598 return CmpInst && MI == CmpInst;
3599 }
3600
3601 std::optional<bool> createTripCountGreaterCondition(
3602 int TC, MachineBasicBlock &MBB,
3603 SmallVectorImpl<MachineOperand> &CondParam) override {
3604 CondParam = this->Cond;
3605 return {};
3606 }
3607
3608 void adjustTripCount(int TripCountAdjust) override {}
3609
3610 void setPreheader(MachineBasicBlock *NewPreheader) override {}
3611};
3612} // namespace
3613
3614std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
3616 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
3618 // Unanalyzable terminator.
3619 if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
3620 return nullptr;
3621
3622 // Infinite loops are not supported.
3623 if (TBB == LoopBB && FBB == LoopBB)
3624 return nullptr;
3625
3626 // Must be conditional branch.
3627 if (FBB == nullptr)
3628 return nullptr;
3629
3630 assert((TBB == LoopBB || FBB == LoopBB) &&
3631 "The Loop must be a single-basic-block loop");
3632
3633 // Divergent (VCC/EXEC) back-edge is not supported.
3634 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3635 if (Pred != SCC_TRUE && Pred != SCC_FALSE)
3636 return nullptr;
3637
3638 // Calls and inline assembly are not supported.
3639 for (const MachineInstr &MI : *LoopBB)
3640 if (MI.isCall() || MI.isInlineAsm())
3641 return nullptr;
3642
3643 // Normalization for createTripCountGreaterCondition(): make Cond mean
3644 // "exit the loop" so the expander emits correct prolog guard branches.
3645 if (TBB == LoopBB)
3647
3648 auto Instructions = make_range(
3650 LoopBB->rend());
3651 auto CmpI = llvm::find_if(Instructions, [&](const MachineInstr &MI) {
3652 return MI.modifiesRegister(Cond[1].getReg(), &RI);
3653 });
3654
3655 if (CmpI == Instructions.end() || CmpI->isPHI())
3656 return nullptr;
3657 MachineInstr *CmpInst = &*CmpI;
3658
3659 return std::make_unique<AMDGPUPipelinerLoopInfo>(CmpInst, Cond);
3660}
3661
3664 Register DstReg, Register TrueReg,
3665 Register FalseReg,
3666 SelectExpansion &Exp) const {
3667 switch (Cond[0].getImm()) {
3668 case VCCNZ:
3669 case VCCZ: {
3670 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3671 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
3672 if (MRI.getRegClass(FalseReg) != RC)
3673 return false;
3674
3675 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;
3676 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
3677
3678 // Limit to equal cost for branch vs. N v_cndmask_b32s.
3679 return RI.hasVGPRs(RC) && NumInsts <= 6;
3680 }
3681 case SCC_TRUE:
3682 case SCC_FALSE: {
3683 // FIXME: We could insert for VGPRs if we could replace the original compare
3684 // with a vector one.
3685 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3686 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
3687 if (MRI.getRegClass(FalseReg) != RC)
3688 return false;
3689
3690 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;
3691
3692 // Multiples of 8 can do s_cselect_b64
3693 if (NumInsts % 2 == 0)
3694 NumInsts /= 2;
3695
3696 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
3697 return RI.isSGPRClass(RC);
3698 }
3699 default:
3700 return false;
3701 }
3702}
3703
3707 Register TrueReg, Register FalseReg) const {
3708 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3709 if (Pred == VCCZ || Pred == SCC_FALSE) {
3710 Pred = static_cast<BranchPredicate>(-Pred);
3711 std::swap(TrueReg, FalseReg);
3712 }
3713
3714 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3715 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
3716 unsigned DstSize = RI.getRegSizeInBits(*DstRC);
3717
3718 if (DstSize == 32) {
3720 if (Pred == SCC_TRUE) {
3721 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg)
3722 .addReg(TrueReg)
3723 .addReg(FalseReg);
3724 } else {
3725 // Instruction's operands are backwards from what is expected.
3726 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg)
3727 .addReg(FalseReg)
3728 .addReg(TrueReg);
3729 }
3730
3731 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
3732 return;
3733 }
3734
3735 if (DstSize == 64 && Pred == SCC_TRUE) {
3737 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
3738 .addReg(TrueReg)
3739 .addReg(FalseReg);
3740
3741 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
3742 return;
3743 }
3744
3745 static const int16_t Sub0_15[] = {
3746 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3747 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3748 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3749 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3750 };
3751
3752 static const int16_t Sub0_15_64[] = {
3753 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3754 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3755 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3756 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3757 };
3758
3759 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3760 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3761 const int16_t *SubIndices = Sub0_15;
3762 int NElts = DstSize / 32;
3763
3764 // 64-bit select is only available for SALU.
3765 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3766 if (Pred == SCC_TRUE) {
3767 if (NElts % 2) {
3768 SelOp = AMDGPU::S_CSELECT_B32;
3769 EltRC = &AMDGPU::SGPR_32RegClass;
3770 } else {
3771 SelOp = AMDGPU::S_CSELECT_B64;
3772 EltRC = &AMDGPU::SGPR_64RegClass;
3773 SubIndices = Sub0_15_64;
3774 NElts /= 2;
3775 }
3776 }
3777
3779 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
3780
3781 I = MIB->getIterator();
3782
3784 for (int Idx = 0; Idx != NElts; ++Idx) {
3785 Register DstElt = MRI.createVirtualRegister(EltRC);
3786 Regs.push_back(DstElt);
3787
3788 unsigned SubIdx = SubIndices[Idx];
3789
3791 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3792 Select = BuildMI(MBB, I, DL, get(SelOp), DstElt)
3793 .addReg(FalseReg, {}, SubIdx)
3794 .addReg(TrueReg, {}, SubIdx);
3795 } else {
3796 Select = BuildMI(MBB, I, DL, get(SelOp), DstElt)
3797 .addReg(TrueReg, {}, SubIdx)
3798 .addReg(FalseReg, {}, SubIdx);
3799 }
3800
3801 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
3803
3804 MIB.addReg(DstElt)
3805 .addImm(SubIdx);
3806 }
3807}
3808
3810
3811 if (MI.isBranch() || MI.isCall() || MI.isReturn() || MI.isIndirectBranch())
3812 return true;
3813
3814 switch (MI.getOpcode()) {
3815 case AMDGPU::S_ENDPGM:
3816 case AMDGPU::S_ENDPGM_SAVED:
3817 case AMDGPU::S_TRAP:
3818 case AMDGPU::S_GETREG_B32:
3819 case AMDGPU::S_SETREG_B32:
3820 case AMDGPU::S_SETREG_B32_mode:
3821 case AMDGPU::S_SETREG_IMM32_B32:
3822 case AMDGPU::S_SETREG_IMM32_B32_mode:
3823 case AMDGPU::S_SENDMSG:
3824 case AMDGPU::S_SENDMSGHALT:
3825 case AMDGPU::S_SENDMSG_RTN_B32:
3826 case AMDGPU::S_SENDMSG_RTN_B64:
3827 case AMDGPU::S_BARRIER_WAIT:
3828 case AMDGPU::S_BARRIER_SIGNAL_M0:
3829 case AMDGPU::S_BARRIER_SIGNAL_IMM:
3830 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
3831 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
3832 return true;
3833 default:
3834 return false;
3835 }
3836}
3837
3839 switch (MI.getOpcode()) {
3840 case AMDGPU::V_MOV_B16_t16_e32:
3841 case AMDGPU::V_MOV_B16_t16_e64:
3842 case AMDGPU::V_MOV_B32_e32:
3843 case AMDGPU::V_MOV_B32_e64:
3844 case AMDGPU::V_MOV_B64_PSEUDO:
3845 case AMDGPU::V_MOV_B64_e32:
3846 case AMDGPU::V_MOV_B64_e64:
3847 case AMDGPU::S_MOV_B32:
3848 case AMDGPU::S_MOV_B64:
3849 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3850 case AMDGPU::COPY:
3851 case AMDGPU::WWM_COPY:
3852 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3853 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3854 case AMDGPU::V_ACCVGPR_MOV_B32:
3855 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3856 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3857 return true;
3858 default:
3859 return false;
3860 }
3861}
3862
3864 switch (MI.getOpcode()) {
3865 case AMDGPU::V_MOV_B16_t16_e32:
3866 case AMDGPU::V_MOV_B16_t16_e64:
3867 return 2;
3868 case AMDGPU::V_MOV_B32_e32:
3869 case AMDGPU::V_MOV_B32_e64:
3870 case AMDGPU::V_MOV_B64_PSEUDO:
3871 case AMDGPU::V_MOV_B64_e32:
3872 case AMDGPU::V_MOV_B64_e64:
3873 case AMDGPU::S_MOV_B32:
3874 case AMDGPU::S_MOV_B64:
3875 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3876 case AMDGPU::COPY:
3877 case AMDGPU::WWM_COPY:
3878 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3879 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3880 case AMDGPU::V_ACCVGPR_MOV_B32:
3881 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3882 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3883 return 1;
3884 default:
3885 llvm_unreachable("MI is not a foldable copy");
3886 }
3887}
3888
3889static constexpr AMDGPU::OpName ModifierOpNames[] = {
3890 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3891 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3892 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3893
3895 unsigned Opc = MI.getOpcode();
3896 for (AMDGPU::OpName Name : reverse(ModifierOpNames)) {
3897 int Idx = AMDGPU::getNamedOperandIdx(Opc, Name);
3898 if (Idx >= 0)
3899 MI.removeOperand(Idx);
3900 }
3901}
3902
3904 const MCInstrDesc &NewDesc) const {
3905 MI.setDesc(NewDesc);
3906
3907 // Remove any leftover implicit operands from mutating the instruction. e.g.
3908 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
3909 // anymore.
3910 const MCInstrDesc &Desc = MI.getDesc();
3911 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
3912 Desc.implicit_defs().size();
3913
3914 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
3915 MI.removeOperand(I);
3916}
3917
3918std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
3919 unsigned SubRegIndex) {
3920 switch (SubRegIndex) {
3921 case AMDGPU::NoSubRegister:
3922 return Imm;
3923 case AMDGPU::sub0:
3924 return SignExtend64<32>(Imm);
3925 case AMDGPU::sub1:
3926 return SignExtend64<32>(Imm >> 32);
3927 case AMDGPU::lo16:
3928 return SignExtend64<16>(Imm);
3929 case AMDGPU::hi16:
3930 return SignExtend64<16>(Imm >> 16);
3931 case AMDGPU::sub1_lo16:
3932 return SignExtend64<16>(Imm >> 32);
3933 case AMDGPU::sub1_hi16:
3934 return SignExtend64<16>(Imm >> 48);
3935 default:
3936 return std::nullopt;
3937 }
3938
3939 llvm_unreachable("covered subregister switch");
3940}
3941
3942std::optional<unsigned>
3944 unsigned SubRegIndex) {
3945 if (SubRegIndex == AMDGPU::NoSubRegister)
3946 return TargetFlags;
3947
3948 unsigned LoReloc, HiReloc;
3949 switch (TargetFlags & MO_MASK) {
3950 case MO_REL32_LO:
3951 case MO_REL32_HI:
3952 case MO_REL64:
3953 LoReloc = MO_REL32_LO;
3954 HiReloc = MO_REL32_HI;
3955 break;
3956 case MO_GOTPCREL32_LO:
3957 case MO_GOTPCREL32_HI:
3958 LoReloc = MO_GOTPCREL32_LO;
3959 HiReloc = MO_GOTPCREL32_HI;
3960 break;
3961 case MO_ABS32_LO:
3962 case MO_ABS32_HI:
3963 case MO_ABS64:
3964 LoReloc = MO_ABS32_LO;
3965 HiReloc = MO_ABS32_HI;
3966 break;
3967 default:
3968 return std::nullopt;
3969 }
3970
3971 const unsigned BaseFlags = TargetFlags & ~MO_MASK;
3972 switch (SubRegIndex) {
3973 case AMDGPU::sub0:
3974 return BaseFlags | LoReloc;
3975 case AMDGPU::sub1:
3976 return BaseFlags | HiReloc;
3977 default:
3978 return std::nullopt;
3979 }
3980}
3981
3982static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) {
3983 switch (Opc) {
3984 case AMDGPU::V_MAC_F16_e32:
3985 case AMDGPU::V_MAC_F16_e64:
3986 case AMDGPU::V_MAD_F16_e64:
3987 return AMDGPU::V_MADAK_F16;
3988 case AMDGPU::V_MAC_F32_e32:
3989 case AMDGPU::V_MAC_F32_e64:
3990 case AMDGPU::V_MAD_F32_e64:
3991 return AMDGPU::V_MADAK_F32;
3992 case AMDGPU::V_FMAC_F32_e32:
3993 case AMDGPU::V_FMAC_F32_e64:
3994 case AMDGPU::V_FMA_F32_e64:
3995 return AMDGPU::V_FMAAK_F32;
3996 case AMDGPU::V_FMAC_F16_e32:
3997 case AMDGPU::V_FMAC_F16_e64:
3998 case AMDGPU::V_FMAC_F16_t16_e64:
3999 case AMDGPU::V_FMAC_F16_fake16_e64:
4000 case AMDGPU::V_FMAC_F16_t16_e32:
4001 case AMDGPU::V_FMAC_F16_fake16_e32:
4002 case AMDGPU::V_FMA_F16_e64:
4003 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4004 ? AMDGPU::V_FMAAK_F16_t16
4005 : AMDGPU::V_FMAAK_F16_fake16
4006 : AMDGPU::V_FMAAK_F16;
4007 case AMDGPU::V_FMAC_F64_e32:
4008 case AMDGPU::V_FMAC_F64_e64:
4009 case AMDGPU::V_FMA_F64_e64:
4010 return AMDGPU::V_FMAAK_F64;
4011 default:
4012 llvm_unreachable("invalid instruction");
4013 }
4014}
4015
4016static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) {
4017 switch (Opc) {
4018 case AMDGPU::V_MAC_F16_e32:
4019 case AMDGPU::V_MAC_F16_e64:
4020 case AMDGPU::V_MAD_F16_e64:
4021 return AMDGPU::V_MADMK_F16;
4022 case AMDGPU::V_MAC_F32_e32:
4023 case AMDGPU::V_MAC_F32_e64:
4024 case AMDGPU::V_MAD_F32_e64:
4025 return AMDGPU::V_MADMK_F32;
4026 case AMDGPU::V_FMAC_F32_e32:
4027 case AMDGPU::V_FMAC_F32_e64:
4028 case AMDGPU::V_FMA_F32_e64:
4029 return AMDGPU::V_FMAMK_F32;
4030 case AMDGPU::V_FMAC_F16_e32:
4031 case AMDGPU::V_FMAC_F16_e64:
4032 case AMDGPU::V_FMAC_F16_t16_e64:
4033 case AMDGPU::V_FMAC_F16_fake16_e64:
4034 case AMDGPU::V_FMAC_F16_t16_e32:
4035 case AMDGPU::V_FMAC_F16_fake16_e32:
4036 case AMDGPU::V_FMA_F16_e64:
4037 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4038 ? AMDGPU::V_FMAMK_F16_t16
4039 : AMDGPU::V_FMAMK_F16_fake16
4040 : AMDGPU::V_FMAMK_F16;
4041 case AMDGPU::V_FMAC_F64_e32:
4042 case AMDGPU::V_FMAC_F64_e64:
4043 case AMDGPU::V_FMA_F64_e64:
4044 return AMDGPU::V_FMAMK_F64;
4045 default:
4046 llvm_unreachable("invalid instruction");
4047 }
4048}
4049
4051 Register Reg, MachineRegisterInfo *MRI) const {
4052 int64_t Imm;
4053 if (!getConstValDefinedInReg(DefMI, Reg, Imm))
4054 return false;
4055
4056 const bool HasMultipleUses = !MRI->hasOneNonDBGUse(Reg);
4057
4058 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
4059
4060 unsigned Opc = UseMI.getOpcode();
4061 if (Opc == AMDGPU::COPY) {
4062 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
4063
4064 Register DstReg = UseMI.getOperand(0).getReg();
4065 Register UseSubReg = UseMI.getOperand(1).getSubReg();
4066
4067 const TargetRegisterClass *DstRC = RI.getRegClassForReg(*MRI, DstReg);
4068
4069 if (HasMultipleUses) {
4070 // TODO: This should fold in more cases with multiple use, but we need to
4071 // more carefully consider what those uses are.
4072 unsigned ImmDefSize = RI.getRegSizeInBits(*MRI->getRegClass(Reg));
4073
4074 // Avoid breaking up a 64-bit inline immediate into a subregister extract.
4075 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)
4076 return false;
4077
4078 // Most of the time folding a 32-bit inline constant is free (though this
4079 // might not be true if we can't later fold it into a real user).
4080 //
4081 // FIXME: This isInlineConstant check is imprecise if
4082 // getConstValDefinedInReg handled the tricky non-mov cases.
4083 if (ImmDefSize == 32 &&
4085 return false;
4086 }
4087
4088 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
4089 RI.getSubRegIdxSize(UseSubReg) == 16;
4090
4091 if (Is16Bit) {
4092 if (RI.hasVGPRs(DstRC))
4093 return false; // Do not clobber vgpr_hi16
4094
4095 if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)
4096 return false;
4097 }
4098
4099 MachineFunction *MF = UseMI.getMF();
4100
4101 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
4102 MCRegister MovDstPhysReg =
4103 DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();
4104
4105 std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, UseSubReg);
4106
4107 // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e64
4108 for (unsigned MovOp :
4109 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
4110 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
4111 const MCInstrDesc &MovDesc = get(MovOp);
4112
4113 const TargetRegisterClass *MovDstRC = getRegClass(MovDesc, 0);
4114 if (Is16Bit) {
4115 // We just need to find a correctly sized register class, so the
4116 // subregister index compatibility doesn't matter since we're statically
4117 // extracting the immediate value.
4118 MovDstRC = RI.getMatchingSuperRegClass(MovDstRC, DstRC, AMDGPU::lo16);
4119 if (!MovDstRC)
4120 continue;
4121
4122 if (MovDstPhysReg) {
4123 // FIXME: We probably should not do this. If there is a live value in
4124 // the high half of the register, it will be corrupted.
4125 MovDstPhysReg =
4126 RI.getMatchingSuperReg(MovDstPhysReg, AMDGPU::lo16, MovDstRC);
4127 if (!MovDstPhysReg)
4128 continue;
4129 }
4130 }
4131
4132 // Result class isn't the right size, try the next instruction.
4133 if (MovDstPhysReg) {
4134 if (!MovDstRC->contains(MovDstPhysReg))
4135 return false;
4136 } else if (!MRI->constrainRegClass(DstReg, MovDstRC)) {
4137 // TODO: This will be overly conservative in the case of 16-bit virtual
4138 // SGPRs. We could hack up the virtual register uses to use a compatible
4139 // 32-bit class.
4140 continue;
4141 }
4142
4143 const MCOperandInfo &OpInfo = MovDesc.operands()[1];
4144
4145 // Ensure the interpreted immediate value is a valid operand in the new
4146 // mov.
4147 //
4148 // FIXME: isImmOperandLegal should have form that doesn't require existing
4149 // MachineInstr or MachineOperand
4150 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType) &&
4151 !isInlineConstant(*SubRegImm, OpInfo.OperandType))
4152 break;
4153
4154 NewOpc = MovOp;
4155 break;
4156 }
4157
4158 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
4159 return false;
4160
4161 if (Is16Bit) {
4162 UseMI.getOperand(0).setSubReg(AMDGPU::NoSubRegister);
4163 if (MovDstPhysReg)
4164 UseMI.getOperand(0).setReg(MovDstPhysReg);
4165 assert(UseMI.getOperand(1).getReg().isVirtual());
4166 }
4167
4168 const MCInstrDesc &NewMCID = get(NewOpc);
4169 UseMI.setDesc(NewMCID);
4170 UseMI.getOperand(1).ChangeToImmediate(*SubRegImm);
4171 UseMI.addImplicitDefUseOperands(*MF);
4172 return true;
4173 }
4174
4175 if (HasMultipleUses)
4176 return false;
4177
4178 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
4179 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4180 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
4181 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
4182 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4183 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 ||
4184 Opc == AMDGPU::V_FMAC_F64_e64) {
4185 // Don't fold if we are using source or output modifiers. The new VOP2
4186 // instructions don't have them.
4188 return false;
4189
4190 // If this is a free constant, there's no reason to do this.
4191 // TODO: We could fold this here instead of letting SIFoldOperands do it
4192 // later.
4193 int Src0Idx = getNamedOperandIdx(UseMI.getOpcode(), AMDGPU::OpName::src0);
4194
4195 // Any src operand can be used for the legality check.
4196 if (isInlineConstant(UseMI, Src0Idx, Imm))
4197 return false;
4198
4199 MachineOperand *Src0 = &UseMI.getOperand(Src0Idx);
4200
4201 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
4202 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
4203
4204 auto CopyRegOperandToNarrowerRC =
4205 [MRI, this](MachineInstr &MI, unsigned OpNo,
4206 const TargetRegisterClass *NewRC) -> void {
4207 if (!MI.getOperand(OpNo).isReg())
4208 return;
4209 Register Reg = MI.getOperand(OpNo).getReg();
4210 const TargetRegisterClass *RC = RI.getRegClassForReg(*MRI, Reg);
4211 if (RI.getCommonSubClass(RC, NewRC) != NewRC)
4212 return;
4213 Register Tmp = MRI->createVirtualRegister(NewRC);
4214 BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(),
4215 get(AMDGPU::COPY), Tmp)
4216 .addReg(Reg);
4217 MI.getOperand(OpNo).setReg(Tmp);
4218 };
4219
4220 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
4221 if ((Src0->isReg() && Src0->getReg() == Reg) ||
4222 (Src1->isReg() && Src1->getReg() == Reg)) {
4223 MachineOperand *RegSrc =
4224 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
4225 if (!RegSrc->isReg())
4226 return false;
4227 if (RI.isSGPRClass(MRI->getRegClass(RegSrc->getReg())) &&
4228 ST.getConstantBusLimit(Opc) < 2)
4229 return false;
4230
4231 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
4232 return false;
4233
4234 // If src2 is also a literal constant then we have to choose which one to
4235 // fold. In general it is better to choose madak so that the other literal
4236 // can be materialized in an sgpr instead of a vgpr:
4237 // s_mov_b32 s0, literal
4238 // v_madak_f32 v0, s0, v0, literal
4239 // Instead of:
4240 // v_mov_b32 v1, literal
4241 // v_madmk_f32 v0, v0, literal, v1
4242 MachineInstr *Def = MRI->getUniqueVRegDef(Src2->getReg());
4243 if (Def && Def->isMoveImmediate() &&
4244 !isInlineConstant(Def->getOperand(1)))
4245 return false;
4246
4247 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4248 if (pseudoToMCOpcode(NewOpc) == -1)
4249 return false;
4250
4251 const std::optional<int64_t> SubRegImm = extractSubregFromImm(
4252 Imm, RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
4253
4254 // FIXME: This would be a lot easier if we could return a new instruction
4255 // instead of having to modify in place.
4256
4257 Register SrcReg = RegSrc->getReg();
4258 unsigned SrcSubReg = RegSrc->getSubReg();
4259 Src0->setReg(SrcReg);
4260 Src0->setSubReg(SrcSubReg);
4261 Src0->setIsKill(RegSrc->isKill());
4262
4263 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4264 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4265 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4266 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4267 UseMI.untieRegOperand(
4268 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
4269
4270 Src1->ChangeToImmediate(*SubRegImm);
4271
4273 UseMI.setDesc(get(NewOpc));
4274
4275 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||
4276 NewOpc == AMDGPU::V_FMAMK_F16_fake16) {
4277 const TargetRegisterClass *NewRC = getRegClass(get(NewOpc), 0);
4278 Register Tmp = MRI->createVirtualRegister(NewRC);
4279 BuildMI(*UseMI.getParent(), std::next(UseMI.getIterator()),
4280 UseMI.getDebugLoc(), get(AMDGPU::COPY),
4281 UseMI.getOperand(0).getReg())
4282 .addReg(Tmp);
4283 UseMI.getOperand(0).setReg(Tmp);
4284 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4285 CopyRegOperandToNarrowerRC(UseMI, 3, NewRC);
4286 }
4287
4288 bool DeleteDef = MRI->use_nodbg_empty(Reg);
4289 if (DeleteDef)
4290 DefMI.eraseFromParent();
4291
4292 return true;
4293 }
4294
4295 // Added part is the constant: Use v_madak_{f16, f32}.
4296 if (Src2->isReg() && Src2->getReg() == Reg) {
4297 if (ST.getConstantBusLimit(Opc) < 2) {
4298 // Not allowed to use constant bus for another operand.
4299 // We can however allow an inline immediate as src0.
4300 bool Src0Inlined = false;
4301 if (Src0->isReg()) {
4302 // Try to inline constant if possible.
4303 // If the Def moves immediate and the use is single
4304 // We are saving VGPR here.
4305 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
4306 if (Def && Def->isMoveImmediate() &&
4307 isInlineConstant(Def->getOperand(1)) &&
4308 MRI->hasOneNonDBGUse(Src0->getReg())) {
4309 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
4310 Src0Inlined = true;
4311 } else if (ST.getConstantBusLimit(Opc) <= 1 &&
4312 RI.isSGPRReg(*MRI, Src0->getReg())) {
4313 return false;
4314 }
4315 // VGPR is okay as Src0 - fallthrough
4316 }
4317
4318 if (Src1->isReg() && !Src0Inlined) {
4319 // We have one slot for inlinable constant so far - try to fill it
4320 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
4321 if (Def && Def->isMoveImmediate() &&
4322 isInlineConstant(Def->getOperand(1)) &&
4323 MRI->hasOneNonDBGUse(Src1->getReg()) && commuteInstruction(UseMI))
4324 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
4325 else if (RI.isSGPRReg(*MRI, Src1->getReg()))
4326 return false;
4327 // VGPR is okay as Src1 - fallthrough
4328 }
4329 }
4330
4331 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4332 if (pseudoToMCOpcode(NewOpc) == -1)
4333 return false;
4334
4335 // FIXME: This would be a lot easier if we could return a new instruction
4336 // instead of having to modify in place.
4337
4338 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4339 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4340 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4341 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4342 UseMI.untieRegOperand(
4343 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
4344
4345 const std::optional<int64_t> SubRegImm =
4346 extractSubregFromImm(Imm, Src2->getSubReg());
4347
4348 // ChangingToImmediate adds Src2 back to the instruction.
4349 Src2->ChangeToImmediate(*SubRegImm);
4350
4351 // These come before src2.
4353 UseMI.setDesc(get(NewOpc));
4354
4355 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||
4356 NewOpc == AMDGPU::V_FMAAK_F16_fake16) {
4357 const TargetRegisterClass *NewRC = getRegClass(get(NewOpc), 0);
4358 Register Tmp = MRI->createVirtualRegister(NewRC);
4359 BuildMI(*UseMI.getParent(), std::next(UseMI.getIterator()),
4360 UseMI.getDebugLoc(), get(AMDGPU::COPY),
4361 UseMI.getOperand(0).getReg())
4362 .addReg(Tmp);
4363 UseMI.getOperand(0).setReg(Tmp);
4364 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4365 CopyRegOperandToNarrowerRC(UseMI, 2, NewRC);
4366 }
4367
4368 // It might happen that UseMI was commuted
4369 // and we now have SGPR as SRC1. If so 2 inlined
4370 // constant and SGPR are illegal.
4372
4373 int NewSrc0Idx =
4374 AMDGPU::getNamedOperandIdx(UseMI.getOpcode(), AMDGPU::OpName::src0);
4375 if (!isOperandLegal(UseMI, NewSrc0Idx))
4376 legalizeOpWithMove(UseMI, NewSrc0Idx);
4377
4378 bool DeleteDef = MRI->use_nodbg_empty(Reg);
4379 if (DeleteDef)
4380 DefMI.eraseFromParent();
4381
4382 return true;
4383 }
4384 }
4385
4386 return false;
4387}
4388
4389static bool
4392 if (BaseOps1.size() != BaseOps2.size())
4393 return false;
4394 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
4395 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I]))
4396 return false;
4397 }
4398 return true;
4399}
4400
4401static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
4402 LocationSize WidthB, int OffsetB) {
4403 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
4404 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
4405 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4406 return LowWidth.hasValue() &&
4407 LowOffset + (int)LowWidth.getValue() <= HighOffset;
4408}
4409
4410bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
4411 const MachineInstr &MIb) const {
4412 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
4413 int64_t Offset0, Offset1;
4414 LocationSize Dummy0 = LocationSize::precise(0);
4415 LocationSize Dummy1 = LocationSize::precise(0);
4416 bool Offset0IsScalable, Offset1IsScalable;
4417 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
4418 Dummy0) ||
4419 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable,
4420 Dummy1))
4421 return false;
4422
4423 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))
4424 return false;
4425
4426 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
4427 // FIXME: Handle ds_read2 / ds_write2.
4428 return false;
4429 }
4430 LocationSize Width0 = MIa.memoperands().front()->getSize();
4431 LocationSize Width1 = MIb.memoperands().front()->getSize();
4432 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);
4433}
4434
4436 const MachineInstr &MIb) const {
4437 assert(MIa.mayLoadOrStore() &&
4438 "MIa must load from or modify a memory location");
4439 assert(MIb.mayLoadOrStore() &&
4440 "MIb must load from or modify a memory location");
4441
4443 return false;
4444
4445 // XXX - Can we relax this between address spaces?
4446 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4447 return false;
4448
4449 if (isLDSDMA(MIa) || isLDSDMA(MIb))
4450 return false;
4451
4452 if (MIa.isBundle() || MIb.isBundle())
4453 return false;
4454
4455 // These only alias each other, and only on overlapping dword ranges.
4456 const bool IsLdStIdxA = isa<AMDGPUMI::VLoadStoreIdxInst>(MIa);
4457 const bool IsLdStIdxB = isa<AMDGPUMI::VLoadStoreIdxInst>(MIb);
4458 if (IsLdStIdxA || IsLdStIdxB) {
4459 if (IsLdStIdxA && IsLdStIdxB)
4460 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4461 return true;
4462 }
4463
4464 // TODO: Should we check the address space from the MachineMemOperand? That
4465 // would allow us to distinguish objects we know don't alias based on the
4466 // underlying address space, even if it was lowered to a different one,
4467 // e.g. private accesses lowered to use MUBUF instructions on a scratch
4468 // buffer.
4469 if (isDS(MIa)) {
4470 if (isDS(MIb))
4471 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4472
4473 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
4474 }
4475
4476 if (isMUBUF(MIa) || isMTBUF(MIa)) {
4477 if (isMUBUF(MIb) || isMTBUF(MIb))
4478 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4479
4480 if (isFLAT(MIb))
4481 return isFLATScratch(MIb);
4482
4483 return !isSMRD(MIb);
4484 }
4485
4486 if (isSMRD(MIa)) {
4487 if (isSMRD(MIb))
4488 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4489
4490 if (isFLAT(MIb))
4491 return isFLATScratch(MIb);
4492
4493 return !isMUBUF(MIb) && !isMTBUF(MIb);
4494 }
4495
4496 if (isFLAT(MIa)) {
4497 if (isFLAT(MIb)) {
4498 if ((isFLATScratch(MIa) && isFLATGlobal(MIb)) ||
4499 (isFLATGlobal(MIa) && isFLATScratch(MIb)))
4500 return true;
4501
4502 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4503 }
4504
4505 return false;
4506 }
4507
4508 return false;
4509}
4510
4511static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc) {
4512 switch (Opc) {
4513 case AMDGPU::V_MAC_F16_e32:
4514 case AMDGPU::V_MAC_F16_e64:
4515 return AMDGPU::V_MAD_F16_e64;
4516 case AMDGPU::V_MAC_F32_e32:
4517 case AMDGPU::V_MAC_F32_e64:
4518 return AMDGPU::V_MAD_F32_e64;
4519 case AMDGPU::V_MAC_LEGACY_F32_e32:
4520 case AMDGPU::V_MAC_LEGACY_F32_e64:
4521 return AMDGPU::V_MAD_LEGACY_F32_e64;
4522 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4523 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4524 return AMDGPU::V_FMA_LEGACY_F32_e64;
4525 case AMDGPU::V_FMAC_F16_e32:
4526 case AMDGPU::V_FMAC_F16_e64:
4527 case AMDGPU::V_FMAC_F16_t16_e64:
4528 case AMDGPU::V_FMAC_F16_fake16_e64:
4529 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4530 ? AMDGPU::V_FMA_F16_gfx9_t16_e64
4531 : AMDGPU::V_FMA_F16_gfx9_fake16_e64
4532 : AMDGPU::V_FMA_F16_gfx9_e64;
4533 case AMDGPU::V_FMAC_F32_e32:
4534 case AMDGPU::V_FMAC_F32_e64:
4535 return AMDGPU::V_FMA_F32_e64;
4536 case AMDGPU::V_FMAC_F64_e32:
4537 case AMDGPU::V_FMAC_F64_e64:
4538 return AMDGPU::V_FMA_F64_e64;
4539 default:
4540 llvm_unreachable("invalid instruction");
4541 }
4542}
4543
4544/// Helper struct for the implementation of 3-address conversion to communicate
4545/// updates made to instruction operands.
4547 /// Other instruction whose def is no longer used by the converted
4548 /// instruction.
4550};
4551
4553 LiveIntervals *LIS) const {
4554 MachineBasicBlock &MBB = *MI.getParent();
4555 MachineInstr *CandidateMI = &MI;
4556
4557 if (MI.isBundle()) {
4558 // This is a temporary placeholder for bundle handling that enables us to
4559 // exercise the relevant code paths in the two-address instruction pass.
4560 if (MI.getBundleSize() != 1)
4561 return nullptr;
4562 CandidateMI = MI.getNextNode();
4563 }
4564
4566 MachineInstr *NewMI = convertToThreeAddressImpl(*CandidateMI, U);
4567 if (!NewMI)
4568 return nullptr;
4569
4570 if (MI.isBundle()) {
4571 CandidateMI->eraseFromBundle();
4572
4573 for (MachineOperand &MO : MI.all_defs()) {
4574 if (MO.isTied())
4575 MI.untieRegOperand(MO.getOperandNo());
4576 }
4577 } else {
4578 if (LIS) {
4579 LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
4580 // SlotIndex of defs needs to be updated when converting to early-clobber
4581 MachineOperand &Def = NewMI->getOperand(0);
4582 if (Def.isEarlyClobber() && Def.isReg() &&
4583 LIS->hasInterval(Def.getReg())) {
4584 SlotIndex OldIndex = LIS->getInstructionIndex(*NewMI).getRegSlot(false);
4585 SlotIndex NewIndex = LIS->getInstructionIndex(*NewMI).getRegSlot(true);
4586 auto &LI = LIS->getInterval(Def.getReg());
4587 auto UpdateDefIndex = [&](LiveRange &LR) {
4588 auto *S = LR.find(OldIndex);
4589 if (S != LR.end() && S->start == OldIndex) {
4590 assert(S->valno && S->valno->def == OldIndex);
4591 S->start = NewIndex;
4592 S->valno->def = NewIndex;
4593 }
4594 };
4595 UpdateDefIndex(LI);
4596 for (auto &SR : LI.subranges())
4597 UpdateDefIndex(SR);
4598 }
4599
4600 // A use tied to an early-clobber def is read at the early-clobber slot.
4601 // The three-address instruction has no tied operands, so the use is now
4602 // read at the register slot.
4603 const MachineOperand &OldDef = MI.getOperand(0);
4604 if (OldDef.isEarlyClobber()) {
4605 assert(OldDef.isTied() &&
4606 "expected a tied def in the two-address form");
4608 MI.getOperand(MI.findTiedOperandIdx(OldDef.getOperandNo()))
4609 .getReg();
4610 SlotIndex Idx = LIS->getInstructionIndex(*NewMI);
4611 SlotIndex OldEnd = Idx.getRegSlot(true);
4612 SlotIndex NewEnd = Idx.getRegSlot();
4613 auto UpdateUseIndex = [&](LiveRange &LR) {
4614 LiveRange::Segment *S = LR.getSegmentContaining(Idx);
4615 if (S && S->end == OldEnd)
4616 S->end = NewEnd;
4617 };
4618 LiveInterval &LI = LIS->getInterval(UseReg);
4619 UpdateUseIndex(LI);
4620 for (LiveInterval::SubRange &SR : LI.subranges())
4621 UpdateUseIndex(SR);
4622 }
4623 }
4624 }
4625
4626 if (U.RemoveMIUse) {
4627 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4628 // The only user is the instruction which will be killed.
4629 Register DefReg = U.RemoveMIUse->getOperand(0).getReg();
4630
4631 if (MRI.hasOneNonDBGUse(DefReg)) {
4632 // We cannot just remove the DefMI here, calling pass will crash.
4633 U.RemoveMIUse->setDesc(get(AMDGPU::IMPLICIT_DEF));
4634 U.RemoveMIUse->getOperand(0).setIsDead(true);
4635 for (unsigned I = U.RemoveMIUse->getNumOperands() - 1; I != 0; --I)
4636 U.RemoveMIUse->removeOperand(I);
4637 }
4638
4639 if (MI.isBundle()) {
4640 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, DefReg);
4641 if (!VRI.Reads && !VRI.Writes) {
4642 for (MachineOperand &MO : MI.all_uses()) {
4643 if (MO.isReg() && MO.getReg() == DefReg) {
4644 assert(MO.getSubReg() == 0 &&
4645 "tied sub-registers in bundles currently not supported");
4646 MI.removeOperand(MO.getOperandNo());
4647 break;
4648 }
4649 }
4650
4651 if (LIS)
4652 LIS->shrinkToUses(&LIS->getInterval(DefReg));
4653 }
4654 } else if (LIS) {
4655 LiveInterval &DefLI = LIS->getInterval(DefReg);
4656
4657 // We cannot delete the original instruction here, so hack out the use
4658 // in the original instruction with a dummy register so we can use
4659 // shrinkToUses to deal with any multi-use edge cases. Other targets do
4660 // not have the complexity of deleting a use to consider here.
4661 Register DummyReg = MRI.cloneVirtualRegister(DefReg);
4662 for (MachineOperand &MIOp : MI.uses()) {
4663 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4664 MIOp.setIsUndef(true);
4665 MIOp.setReg(DummyReg);
4666 }
4667 }
4668
4669 if (MI.isBundle()) {
4670 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, DefReg);
4671 if (!VRI.Reads && !VRI.Writes) {
4672 for (MachineOperand &MIOp : MI.uses()) {
4673 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4674 MIOp.setIsUndef(true);
4675 MIOp.setReg(DummyReg);
4676 }
4677 }
4678 }
4679
4680 MI.addOperand(MachineOperand::CreateReg(DummyReg, false, false, false,
4681 false, /*isUndef=*/true));
4682 }
4683
4684 LIS->shrinkToUses(&DefLI);
4685 }
4686 }
4687
4688 return MI.isBundle() ? &MI : NewMI;
4689}
4690
4692SIInstrInfo::convertToThreeAddressImpl(MachineInstr &MI,
4693 ThreeAddressUpdates &U) const {
4694 MachineBasicBlock &MBB = *MI.getParent();
4695 unsigned Opc = MI.getOpcode();
4696
4697 // Handle MFMA.
4698 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc);
4699 if (NewMFMAOpc != -1) {
4701 BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc));
4702 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4703 MIB.add(MI.getOperand(I));
4704 return MIB;
4705 }
4706
4707 if (SIInstrInfo::isWMMA(MI)) {
4708 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode());
4709 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
4710 .setMIFlags(MI.getFlags());
4711 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4712 MIB->addOperand(MI.getOperand(I));
4713 return MIB;
4714 }
4715
4716 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
4717 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
4718 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "
4719 "present pre-RA");
4720
4721 // Handle MAC/FMAC.
4722 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
4723 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
4724 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
4725 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
4726 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
4727 bool Src0Literal = false;
4728
4729 switch (Opc) {
4730 default:
4731 return nullptr;
4732 case AMDGPU::V_MAC_F16_e64:
4733 case AMDGPU::V_FMAC_F16_e64:
4734 case AMDGPU::V_FMAC_F16_t16_e64:
4735 case AMDGPU::V_FMAC_F16_fake16_e64:
4736 case AMDGPU::V_MAC_F32_e64:
4737 case AMDGPU::V_MAC_LEGACY_F32_e64:
4738 case AMDGPU::V_FMAC_F32_e64:
4739 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4740 case AMDGPU::V_FMAC_F64_e64:
4741 break;
4742 case AMDGPU::V_MAC_F16_e32:
4743 case AMDGPU::V_FMAC_F16_e32:
4744 case AMDGPU::V_MAC_F32_e32:
4745 case AMDGPU::V_MAC_LEGACY_F32_e32:
4746 case AMDGPU::V_FMAC_F32_e32:
4747 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4748 case AMDGPU::V_FMAC_F64_e32: {
4749 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
4750 AMDGPU::OpName::src0);
4751 const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
4752 if (!Src0->isReg() && !Src0->isImm())
4753 return nullptr;
4754
4755 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
4756 Src0Literal = true;
4757
4758 break;
4759 }
4760 }
4761
4762 MachineInstrBuilder MIB;
4763 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4764 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
4765 const MachineOperand *Src0Mods =
4766 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
4767 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
4768 const MachineOperand *Src1Mods =
4769 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
4770 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
4771 const MachineOperand *Src2Mods =
4772 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers);
4773 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
4774 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
4775 const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel);
4776
4777 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&
4778 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&
4779 // If we have an SGPR input, we will violate the constant bus restriction.
4780 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() ||
4781 !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) {
4782 MachineInstr *DefMI = nullptr;
4783 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4784 std::optional<int64_t> ImmOpt;
4785 int64_t Imm;
4786
4787 if (!Src0Literal &&
4788 (ImmOpt = getImmOrMaterializedImm(MRI, *Src2, &DefMI))) {
4789 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4790 if (pseudoToMCOpcode(NewOpc) != -1) {
4791 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
4792 .add(*Dst)
4793 .add(*Src0)
4794 .add(*Src1)
4795 .addImm(*ImmOpt)
4796 .setMIFlags(MI.getFlags());
4797 U.RemoveMIUse = DefMI;
4798 return MIB;
4799 }
4800 }
4801 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4802 if (!Src0Literal &&
4803 (ImmOpt = getImmOrMaterializedImm(MRI, *Src1, &DefMI))) {
4804 if (pseudoToMCOpcode(NewOpc) != -1) {
4805 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
4806 .add(*Dst)
4807 .add(*Src0)
4808 .addImm(*ImmOpt)
4809 .add(*Src2)
4810 .setMIFlags(MI.getFlags());
4811 U.RemoveMIUse = DefMI;
4812 return MIB;
4813 }
4814 }
4815 if ((ImmOpt = getImmOrMaterializedImm(MRI, *Src0, &DefMI))) {
4816 Imm = *ImmOpt;
4817 if (pseudoToMCOpcode(NewOpc) != -1 &&
4819 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
4820 Src1)) {
4821 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
4822 .add(*Dst)
4823 .add(*Src1)
4824 .addImm(Imm)
4825 .add(*Src2)
4826 .setMIFlags(MI.getFlags());
4827 U.RemoveMIUse = DefMI;
4828 return MIB;
4829 }
4830 }
4831 }
4832
4833 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
4834 // if VOP3 does not allow a literal operand.
4835 if (Src0Literal && !ST.hasVOP3Literal())
4836 return nullptr;
4837
4838 unsigned NewOpc = getNewFMAInst(ST, Opc);
4839
4840 if (pseudoToMCOpcode(NewOpc) == -1)
4841 return nullptr;
4842
4843 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
4844 .add(*Dst)
4845 .addImm(Src0Mods ? Src0Mods->getImm() : 0)
4846 .add(*Src0)
4847 .addImm(Src1Mods ? Src1Mods->getImm() : 0)
4848 .add(*Src1)
4849 .addImm(Src2Mods ? Src2Mods->getImm() : 0)
4850 .add(*Src2)
4851 .addImm(Clamp ? Clamp->getImm() : 0)
4852 .addImm(Omod ? Omod->getImm() : 0)
4853 .setMIFlags(MI.getFlags());
4854 if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel))
4855 MIB.addImm(OpSel ? OpSel->getImm() : 0);
4856 return MIB;
4857}
4858
4859// It's not generally safe to move VALU instructions across these since it will
4860// start using the register as a base index rather than directly.
4861// XXX - Why isn't hasSideEffects sufficient for these?
4863 switch (MI.getOpcode()) {
4864 case AMDGPU::S_SET_GPR_IDX_ON:
4865 case AMDGPU::S_SET_GPR_IDX_MODE:
4866 case AMDGPU::S_SET_GPR_IDX_OFF:
4867 return true;
4868 default:
4869 return false;
4870 }
4871}
4872
4874 const MachineBasicBlock *MBB,
4875 const MachineFunction &MF) const {
4876 // Skipping the check for SP writes in the base implementation. The reason it
4877 // was added was apparently due to compile time concerns.
4878 //
4879 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4880 // but is probably avoidable.
4881
4882 // Copied from base implementation.
4883 // Terminators and labels can't be scheduled around.
4884 if (MI.isTerminator() || MI.isPosition())
4885 return true;
4886
4887 // INLINEASM_BR can jump to another block
4888 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4889 return true;
4890
4891 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0)
4892 return true;
4893
4894 // Target-independent instructions do not have an implicit-use of EXEC, even
4895 // when they operate on VGPRs. Treating EXEC modifications as scheduling
4896 // boundaries prevents incorrect movements of such instructions.
4897 return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
4898 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4899 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4900 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4901 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
4903}
4904
4906 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
4907 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
4908 Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode);
4909}
4910
4912 // Instructions that access scratch use FLAT encoding or BUF encodings.
4913 if ((!isFLAT(MI) || isFLATGlobal(MI)) && !isBUF(MI))
4914 return false;
4915
4916 // SCRATCH instructions always access scratch.
4917 if (isFLATScratch(MI))
4918 return true;
4919
4920 // If FLAT_SCRATCH registers are not initialized, we can never access scratch
4921 // via the aperture.
4922 if (MI.getMF()->getFunction().hasFnAttribute("amdgpu-no-flat-scratch-init"))
4923 return false;
4924
4925 // If there are no memory operands then conservatively assume the flat
4926 // operation may access scratch.
4927 if (MI.memoperands_empty())
4928 return true;
4929
4930 // See if any memory operand specifies an address space that involves scratch.
4931 return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) {
4932 unsigned AS = Memop->getAddrSpace();
4933 if (AS == AMDGPUAS::FLAT_ADDRESS) {
4934 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
4935 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
4936 *MD, AMDGPUAS::PRIVATE_ADDRESS);
4937 }
4938 return AS == AMDGPUAS::PRIVATE_ADDRESS;
4939 });
4940}
4941
4943 assert(isFLAT(MI));
4944
4945 // All flat instructions use the VMEM counter except prefetch.
4946 if (!usesVM_CNT(MI))
4947 return false;
4948
4949 // If there are no memory operands then conservatively assume the flat
4950 // operation may access VMEM.
4951 if (MI.memoperands_empty())
4952 return true;
4953
4954 // See if any memory operand specifies an address space that involves VMEM.
4955 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces
4956 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION
4957 // (GDS) address space is not supported by flat operations. Therefore, simply
4958 // return true unless only the LDS address space is found.
4959 for (const MachineMemOperand *Memop : MI.memoperands()) {
4960 unsigned AS = Memop->getAddrSpace();
4962 if (AS != AMDGPUAS::LOCAL_ADDRESS)
4963 return true;
4964 }
4965
4966 return false;
4967}
4968
4970 bool TgSplit) const {
4971 assert(isFLAT(MI));
4972
4973 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.
4974 if (!usesLGKM_CNT(MI))
4975 return false;
4976
4977 // If in tgsplit mode then there can be no use of LDS.
4978 if (TgSplit)
4979 return false;
4980
4981 // If there are no memory operands then conservatively assume the flat
4982 // operation may access LDS.
4983 if (MI.memoperands_empty())
4984 return true;
4985
4986 // See if any memory operand specifies an address space that involves LDS.
4987 for (const MachineMemOperand *Memop : MI.memoperands()) {
4988 unsigned AS = Memop->getAddrSpace();
4990 return true;
4991 }
4992
4993 return false;
4994}
4995
4997 if (!isAsync(MI))
4999 if (usesTENSOR_CNT(MI))
5001 if (!usesASYNC_CNT(MI))
5003
5004 // TODO: Declare the stage of each async instruction in TableGen alongside
5005 // its definition, instead of listing opcodes here.
5006 switch (MI.getOpcode()) {
5007 // Keep in sync with FLATInstructions.td.
5008 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8:
5009 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay:
5010 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_SADDR:
5011 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32:
5012 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR:
5013 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64:
5014 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR:
5015 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128:
5016 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR:
5018 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8:
5019 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8_SADDR:
5020 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32:
5021 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR:
5022 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64:
5023 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64_SADDR:
5024 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128:
5025 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128_SADDR:
5027 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8:
5028 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR:
5029 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32:
5030 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR:
5031 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64:
5032 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR:
5033 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128:
5034 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR:
5036 default:
5037 llvm_unreachable("Async opcode has no associated async stage");
5038 }
5039}
5040
5042 // Skip the full operand and register alias search modifiesRegister
5043 // does. There's only a handful of instructions that touch this, it's only an
5044 // implicit def, and doesn't alias any other registers.
5045 return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE);
5046}
5047
5049 unsigned Opcode = MI.getOpcode();
5050
5051 if (MI.mayStore() && isSMRD(MI))
5052 return true; // scalar store or atomic
5053
5054 // This will terminate the function when other lanes may need to continue.
5055 if (MI.isReturn())
5056 return true;
5057
5058 // These instructions cause shader I/O that may cause hardware lockups
5059 // when executed with an empty EXEC mask.
5060 //
5061 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
5062 // EXEC = 0, but checking for that case here seems not worth it
5063 // given the typical code patterns.
5064 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
5065 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
5066 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT ||
5067 Opcode == AMDGPU::S_SETHALT)
5068 return true;
5069
5070 if (MI.isCall() || MI.isInlineAsm())
5071 return true; // conservative assumption
5072
5073 // V_PERM_PK16 must issue with EXEC != 0 so its follower (or an inserted
5074 // V_NOP) actually runs on the VALU pipe. Returning true here keeps the
5075 // s_cbranch_execz that skips this region when EXEC is empty.
5076 if (ST.hasVPermPk16Hazard() && isVPermPk16(Opcode))
5077 return true;
5078
5079 // Assume that barrier interactions are only intended with active lanes.
5080 if (isBarrier(Opcode))
5081 return true;
5082
5083 // A mode change is a scalar operation that influences vector instructions.
5085 return true;
5086
5087 // These are like SALU instructions in terms of effects, so it's questionable
5088 // whether we should return true for those.
5089 //
5090 // However, executing them with EXEC = 0 causes them to operate on undefined
5091 // data, which we avoid by returning true here.
5092 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
5093 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
5094 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
5095 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
5096 return true;
5097
5098 return false;
5099}
5100
5102 const MachineInstr &MI) const {
5103 if (MI.isMetaInstruction())
5104 return false;
5105
5106 // This won't read exec if this is an SGPR->SGPR copy.
5107 if (MI.isCopyLike()) {
5108 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
5109 return true;
5110
5111 // Make sure this isn't copying exec as a normal operand
5112 return MI.readsRegister(AMDGPU::EXEC, &RI);
5113 }
5114
5115 // Make a conservative assumption about the callee.
5116 if (MI.isCall())
5117 return true;
5118
5119 // Be conservative with any unhandled generic opcodes.
5120 if (!isTargetSpecificOpcode(MI.getOpcode()))
5121 return true;
5122
5123 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
5124}
5125
5127 switch (Imm.getBitWidth()) {
5128 case 1: // This likely will be a condition code mask.
5129 return true;
5130
5131 case 32:
5132 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
5133 ST.hasInv2PiInlineImm());
5134 case 64:
5135 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
5136 ST.hasInv2PiInlineImm());
5137 case 16:
5138 return ST.has16BitInsts() &&
5139 AMDGPU::isInlinableLiteralI16(Imm.getSExtValue(),
5140 ST.hasInv2PiInlineImm());
5141 default:
5142 llvm_unreachable("invalid bitwidth");
5143 }
5144}
5145
5147 APInt IntImm = Imm.bitcastToAPInt();
5148 int64_t IntImmVal = IntImm.getSExtValue();
5149 bool HasInv2Pi = ST.hasInv2PiInlineImm();
5150 switch (APFloat::SemanticsToEnum(Imm.getSemantics())) {
5151 default:
5152 llvm_unreachable("invalid fltSemantics");
5155 return isInlineConstant(IntImm);
5157 return ST.has16BitInsts() &&
5158 AMDGPU::isInlinableLiteralBF16(IntImmVal, HasInv2Pi);
5160 return ST.has16BitInsts() &&
5161 AMDGPU::isInlinableLiteralFP16(IntImmVal, HasInv2Pi);
5162 }
5163}
5164
5165bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {
5166 // MachineOperand provides no way to tell the true operand size, since it only
5167 // records a 64-bit value. We need to know the size to determine if a 32-bit
5168 // floating point immediate bit pattern is legal for an integer immediate. It
5169 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
5170 switch (OperandType) {
5180 int32_t Trunc = static_cast<int32_t>(Imm);
5181 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
5182 }
5190 return AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm());
5193 // We would expect inline immediates to not be concerned with an integer/fp
5194 // distinction. However, in the case of 16-bit integer operations, the
5195 // "floating point" values appear to not work. It seems read the low 16-bits
5196 // of 32-bit immediates, which happens to always work for the integer
5197 // values.
5198 //
5199 // See llvm bugzilla 46302.
5200 //
5201 // TODO: Theoretically we could use op-sel to use the high bits of the
5202 // 32-bit FP values.
5211 return AMDGPU::isPKFMACF16InlineConstant(Imm, ST.isGFX11Plus());
5216 return false;
5219 if (isInt<16>(Imm) || isUInt<16>(Imm)) {
5220 // A few special case instructions have 16-bit operands on subtargets
5221 // where 16-bit instructions are not legal.
5222 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
5223 // constants in these cases
5224 int16_t Trunc = static_cast<int16_t>(Imm);
5225 return ST.has16BitInsts() &&
5226 AMDGPU::isInlinableLiteralFP16(Trunc, ST.hasInv2PiInlineImm());
5227 }
5228
5229 return false;
5230 }
5233 if (isInt<16>(Imm) || isUInt<16>(Imm)) {
5234 int16_t Trunc = static_cast<int16_t>(Imm);
5235 return ST.has16BitInsts() &&
5236 AMDGPU::isInlinableLiteralBF16(Trunc, ST.hasInv2PiInlineImm());
5237 }
5238 return false;
5239 }
5244 return false;
5246 return isLegalAV64PseudoImm(Imm);
5249 // Always embedded in the instruction for free.
5250 return true;
5260 // Just ignore anything else.
5261 return false;
5262 default:
5263 llvm_unreachable("invalid operand type");
5264 }
5265}
5266
5267static bool compareMachineOp(const MachineOperand &Op0,
5268 const MachineOperand &Op1) {
5269 if (Op0.getType() != Op1.getType())
5270 return false;
5271
5272 switch (Op0.getType()) {
5274 return Op0.getReg() == Op1.getReg();
5276 return Op0.getImm() == Op1.getImm();
5277 default:
5278 llvm_unreachable("Didn't expect to be comparing these operand types");
5279 }
5280}
5281
5283 const MCOperandInfo &OpInfo) const {
5284 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
5285 return true;
5286
5287 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
5288 return false;
5289
5290 if (!isVOP3(InstDesc) || !AMDGPU::isSISrcOperand(OpInfo))
5291 return true;
5292
5293 return ST.hasVOP3Literal();
5294}
5295
5296bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5297 int64_t ImmVal) const {
5298 const unsigned Opc = InstDesc.getOpcode();
5299 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
5300 if (Src1Idx != -1 && isDPP(Opc) && !ST.hasDPPSrc1SGPR() &&
5301 OpNo == static_cast<unsigned>(Src1Idx))
5302 return false;
5303
5304 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5305 if (isInlineConstant(ImmVal, OpInfo.OperandType)) {
5306 if (isMAI(InstDesc) && ST.hasMFMAInlineLiteralBug() &&
5307 OpNo == (unsigned)AMDGPU::getNamedOperandIdx(InstDesc.getOpcode(),
5308 AMDGPU::OpName::src2))
5309 return false;
5310
5311 if (ST.hasBF16InlineConstFromUpperFP32() && isVOP1(Opc)) {
5312 if ((OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
5313 OpInfo.OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
5314 isInlineConstant(ImmVal, OpInfo.OperandType))
5315 return false;
5316 }
5317
5318 return RI.opCanUseInlineConstant(OpInfo.OperandType);
5319 }
5320
5321 return isLiteralOperandLegal(InstDesc, OpInfo);
5322}
5323
5324bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5325 const MachineOperand &MO) const {
5326 if (MO.isImm())
5327 return isImmOperandLegal(InstDesc, OpNo, MO.getImm());
5328
5329 assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) &&
5330 "unexpected imm-like operand kind");
5331 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5332 return isLiteralOperandLegal(InstDesc, OpInfo);
5333}
5334
5336 // 2 32-bit inline constants packed into one.
5337 return AMDGPU::isInlinableLiteral32(Lo_32(Imm), ST.hasInv2PiInlineImm()) &&
5338 AMDGPU::isInlinableLiteral32(Hi_32(Imm), ST.hasInv2PiInlineImm());
5339}
5340
5341bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
5342 // GFX90A does not have V_MUL_LEGACY_F32_e32.
5343 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
5344 return false;
5345
5346 int Op32 = AMDGPU::getVOPe32(Opcode);
5347 if (Op32 == -1)
5348 return false;
5349
5350 return pseudoToMCOpcode(Op32) != -1;
5351}
5352
5353/// Return true if \p MI is a VALU comparison, i.e. an instruction that writes
5354/// a lane mask with one bit per lane, and zeroes the bits of lanes that were
5355/// inactive when it executed.
5356///
5357/// TODO: Also handle the sdst result of V_ADD_CO_U32 and V_SUB_CO_U32 and
5358/// V_DIV_SCALE_F32.
5359static bool isVCmp(const SIInstrInfo &TII, const MachineInstr &MI) {
5360 if (TII.isVOPC(MI))
5361 return true;
5362 int Op32 = AMDGPU::getVOPe32(MI.getOpcode());
5363 return Op32 != -1 && TII.isVOPC(Op32);
5364}
5365
5367 const MachineRegisterInfo &MRI,
5368 unsigned Depth) const {
5369 assert(MRI.isSSA() && "isMaskedByExec requires SSA form");
5371 const MachineBasicBlock *MBB = Use.getParent();
5372
5373 // EXEC itself is trivially masked by EXEC.
5374 if (Reg == LMC.ExecReg)
5375 return true;
5376
5377 // Maximum depth of the def-use walk.
5378 constexpr unsigned MaxDepth = 6;
5379 if (Depth >= MaxDepth || !Reg.isVirtual())
5380 return false;
5381
5382 // Only look at definitions that can execute under the same EXEC mask as the
5383 // use.
5384 const MachineInstr *Def = MRI.getVRegDef(Reg);
5385 if (!Def || Def->getParent() != MBB)
5386 return false;
5387
5388 if (isVCmp(*this, *Def))
5389 return true;
5390
5391 // Recurse into an operand, which must be a whole register to say anything
5392 // about the whole lane mask.
5393 auto Recurse = [&](unsigned OpIdx) {
5394 const MachineOperand &MO = Def->getOperand(OpIdx);
5395 return MO.isReg() && !MO.getSubReg() &&
5396 isMaskedByExec(MO.getReg(), Use, MRI, Depth + 1);
5397 };
5398
5399 unsigned Opc = Def->getOpcode();
5400 if (Opc == AMDGPU::COPY && Recurse(1))
5401 return true;
5402 if (Opc == LMC.AndOpc && (Recurse(1) || Recurse(2)))
5403 return true;
5404 if (Opc == LMC.AndN2Opc && Recurse(1))
5405 return true;
5406 if ((Opc == LMC.OrOpc || Opc == LMC.XorOpc) && Recurse(1) && Recurse(2))
5407 return true;
5408 // TODO: Sometimes we encounter "reg = S_CSELECT -1, 0". If Reg has no other
5409 // uses this could be optimized to "reg = S_CSELECT $exec, 0".
5410
5411 return false;
5412}
5413
5414bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
5415 // The src0_modifier operand is present on all instructions
5416 // that have modifiers.
5417
5418 return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers);
5419}
5420
5422 AMDGPU::OpName OpName) const {
5423 const MachineOperand *Mods = getNamedOperand(MI, OpName);
5424 return Mods && Mods->getImm();
5425}
5426
5428 return any_of(ModifierOpNames,
5429 [&](AMDGPU::OpName Name) { return hasModifiersSet(MI, Name); });
5430}
5431
5433 const MachineRegisterInfo &MRI) const {
5434 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
5435 // Can't shrink instruction with three operands.
5436 if (Src2) {
5437 switch (MI.getOpcode()) {
5438 default: return false;
5439
5440 case AMDGPU::V_ADDC_U32_e64:
5441 case AMDGPU::V_SUBB_U32_e64:
5442 case AMDGPU::V_SUBBREV_U32_e64: {
5443 const MachineOperand *Src1
5444 = getNamedOperand(MI, AMDGPU::OpName::src1);
5445 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
5446 return false;
5447 // Additional verification is needed for sdst/src2.
5448 return true;
5449 }
5450 case AMDGPU::V_MAC_F16_e64:
5451 case AMDGPU::V_MAC_F32_e64:
5452 case AMDGPU::V_MAC_LEGACY_F32_e64:
5453 case AMDGPU::V_FMAC_F16_e64:
5454 case AMDGPU::V_FMAC_F16_t16_e64:
5455 case AMDGPU::V_FMAC_F16_fake16_e64:
5456 case AMDGPU::V_FMAC_F32_e64:
5457 case AMDGPU::V_FMAC_F64_e64:
5458 case AMDGPU::V_FMAC_LEGACY_F32_e64:
5459 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
5460 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
5461 return false;
5462 break;
5463
5464 case AMDGPU::V_CNDMASK_B32_e64:
5465 break;
5466 }
5467 }
5468
5469 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
5470 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
5471 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
5472 return false;
5473
5474 // Make sure src0 isn't using any modifiers.
5475 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
5476 return false;
5477
5478 // Can it be shrunk to a valid 32 bit opcode?
5479 if (!hasVALU32BitEncoding(MI.getOpcode()))
5480 return false;
5481
5482 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
5483 if (Src0 && Src0->isImm()) {
5484 unsigned Op32 = AMDGPU::getVOPe32(MI.getOpcode());
5485 if (!isImmOperandLegal(
5486 get(Op32), AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src0),
5487 *Src0))
5488 return false;
5489 }
5490
5491 // Check output modifiers
5492 return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
5493 !hasModifiersSet(MI, AMDGPU::OpName::clamp) &&
5494 !hasModifiersSet(MI, AMDGPU::OpName::byte_sel) &&
5495 // TODO: Can we avoid checking bound_ctrl/fi here?
5496 // They are only used by permlane*_swap special case.
5497 !hasModifiersSet(MI, AMDGPU::OpName::bound_ctrl) &&
5498 !hasModifiersSet(MI, AMDGPU::OpName::fi);
5499}
5500
5501// Set VCC operand with all flags from \p Orig, except for setting it as
5502// implicit.
5504 const MachineOperand &Orig) {
5505
5506 for (MachineOperand &Use : MI.implicit_operands()) {
5507 if (Use.isUse() &&
5508 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
5509 Use.setIsUndef(Orig.isUndef());
5510 Use.setIsKill(Orig.isKill());
5511 return;
5512 }
5513 }
5514}
5515
5517 unsigned Op32) const {
5518 MachineBasicBlock *MBB = MI.getParent();
5519
5520 const MCInstrDesc &Op32Desc = get(Op32);
5521 MachineInstrBuilder Inst32 =
5522 BuildMI(*MBB, MI, MI.getDebugLoc(), Op32Desc)
5523 .setMIFlags(MI.getFlags());
5524
5525 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
5526 // For VOPC instructions, this is replaced by an implicit def of vcc.
5527
5528 // We assume the defs of the shrunk opcode are in the same order, and the
5529 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).
5530 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)
5531 Inst32.add(MI.getOperand(I));
5532
5533 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
5534
5535 int Idx = MI.getNumExplicitDefs();
5536 for (const MachineOperand &Use : MI.explicit_uses()) {
5537 int OpTy = MI.getDesc().operands()[Idx++].OperandType;
5539 continue;
5540
5541 if (&Use == Src2) {
5542 if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2) == -1) {
5543 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
5544 // replaced with an implicit read of vcc or vcc_lo. The implicit read
5545 // of vcc was already added during the initial BuildMI, but we
5546 // 1) may need to change vcc to vcc_lo to preserve the original register
5547 // 2) have to preserve the original flags.
5548 copyFlagsToImplicitVCC(*Inst32, *Src2);
5549 continue;
5550 }
5551 }
5552
5553 Inst32.add(Use);
5554 }
5555
5556 // FIXME: Losing implicit operands
5557 fixImplicitOperands(*Inst32);
5558
5559 // The explicit carry/result def is dropped in favor of an implicit VCC def;
5560 // preserve the dead flag.
5561 const MachineOperand *OldSDst = getNamedOperand(MI, AMDGPU::OpName::sdst);
5562 if (OldSDst && OldSDst->isDead()) {
5563 if (MachineOperand *NewVCC =
5564 Inst32->findRegisterDefOperand(RI.getVCC(), &RI))
5565 NewVCC->setIsDead();
5566 }
5567
5568 return Inst32;
5569}
5570
5572 // Null is free
5573 Register Reg = RegOp.getReg();
5574 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)
5575 return false;
5576
5577 // SGPRs use the constant bus
5578
5579 // FIXME: implicit registers that are not part of the MCInstrDesc's implicit
5580 // physical register operands should also count, except for exec.
5581 if (RegOp.isImplicit())
5582 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;
5583
5584 // SGPRs use the constant bus
5585 return AMDGPU::SReg_32RegClass.contains(Reg) ||
5586 AMDGPU::SReg_64RegClass.contains(Reg);
5587}
5588
5590 const MachineRegisterInfo &MRI) const {
5591 Register Reg = RegOp.getReg();
5592 return Reg.isVirtual() ? RI.isSGPRClass(MRI.getRegClass(Reg))
5593 : physRegUsesConstantBus(RegOp);
5594}
5595
5597 const MachineOperand &MO,
5598 const MCOperandInfo &OpInfo) const {
5599 // Literal constants use the constant bus.
5600 if (!MO.isReg())
5601 return !isInlineConstant(MO, OpInfo);
5602
5603 Register Reg = MO.getReg();
5604 return Reg.isVirtual() ? RI.isSGPRClass(MRI.getRegClass(Reg))
5606}
5607
5609 for (const MachineOperand &MO : MI.implicit_operands()) {
5610 // We only care about reads.
5611 if (MO.isDef())
5612 continue;
5613
5614 switch (MO.getReg()) {
5615 case AMDGPU::VCC:
5616 case AMDGPU::VCC_LO:
5617 case AMDGPU::VCC_HI:
5618 case AMDGPU::M0:
5619 case AMDGPU::FLAT_SCR:
5620 return MO.getReg();
5621
5622 default:
5623 break;
5624 }
5625 }
5626
5627 return Register();
5628}
5629
5630static bool shouldReadExec(const MachineInstr &MI) {
5631 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false)) {
5632 switch (MI.getOpcode()) {
5633 case AMDGPU::V_READLANE_B32:
5634 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
5635 case AMDGPU::V_WRITELANE_B32:
5636 case AMDGPU::SI_SPILL_S32_TO_VGPR:
5637 return false;
5638 }
5639
5640 return true;
5641 }
5642
5643 if (MI.isPreISelOpcode() ||
5644 SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
5647 return false;
5648
5649 return true;
5650}
5651
5652static bool isRegOrFI(const MachineOperand &MO) {
5653 return MO.isReg() || MO.isFI();
5654}
5655
5656static bool isSubRegOf(const SIRegisterInfo &TRI,
5657 const MachineOperand &SuperVec,
5658 const MachineOperand &SubReg) {
5659 if (SubReg.getReg().isPhysical())
5660 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
5661
5662 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
5663 SubReg.getReg() == SuperVec.getReg();
5664}
5665
5666// Verify the illegal copy from vector register to SGPR for generic opcode COPY
5667bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
5668 const MachineRegisterInfo &MRI,
5669 StringRef &ErrInfo) const {
5670 Register DstReg = MI.getOperand(0).getReg();
5671 Register SrcReg = MI.getOperand(1).getReg();
5672 // This is a check for copy from vector register to SGPR
5673 if (RI.isVectorRegister(MRI, SrcReg) && RI.isSGPRReg(MRI, DstReg)) {
5674 ErrInfo = "illegal copy from vector register to SGPR";
5675 return false;
5676 }
5677 return true;
5678}
5679
5681 StringRef &ErrInfo) const {
5682 uint32_t Opcode = MI.getOpcode();
5683 const MachineFunction *MF = MI.getMF();
5684 const MachineRegisterInfo &MRI = MF->getRegInfo();
5685
5686 // FIXME: At this point the COPY verify is done only for non-ssa forms.
5687 // Find a better property to recognize the point where instruction selection
5688 // is just done.
5689 // We can only enforce this check after SIFixSGPRCopies pass so that the
5690 // illegal copies are legalized and thereafter we don't expect a pass
5691 // inserting similar copies.
5692 if (!MRI.isSSA() && MI.isCopy())
5693 return verifyCopy(MI, MRI, ErrInfo);
5694
5695 if (SIInstrInfo::isGenericOpcode(Opcode) && !MI.isInlineAsm())
5696 return true;
5697
5698 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
5699 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
5700 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
5701 int Src3Idx = -1;
5702 if (Src0Idx == -1) {
5703 // VOPD V_DUAL_* instructions use different operand names.
5704 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X);
5705 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X);
5706 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y);
5707 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y);
5708 }
5709
5710 // Make sure the number of operands is correct.
5711 const MCInstrDesc &Desc = get(Opcode);
5712 if (!Desc.isVariadic() &&
5713 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
5714 ErrInfo = "Instruction has wrong number of operands.";
5715 return false;
5716 }
5717
5718 if (MI.isInlineAsm()) {
5719 // Verify register classes for inlineasm constraints.
5720 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
5721 I != E; ++I) {
5722 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
5723 if (!RC)
5724 continue;
5725
5726 const MachineOperand &Op = MI.getOperand(I);
5727 if (!Op.isReg())
5728 continue;
5729
5730 Register Reg = Op.getReg();
5731 if (!Reg.isVirtual() && !RC->contains(Reg)) {
5732 ErrInfo = "inlineasm operand has incorrect register class.";
5733 return false;
5734 }
5735 }
5736
5737 return true;
5738 }
5739
5740 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
5741 ErrInfo = "missing memory operand from image instruction.";
5742 return false;
5743 }
5744
5745 // Make sure the register classes are correct.
5746 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
5747 const MachineOperand &MO = MI.getOperand(i);
5748 if (MO.isFPImm()) {
5749 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
5750 "all fp values to integers.";
5751 return false;
5752 }
5753
5754 const MCOperandInfo &OpInfo = Desc.operands()[i];
5755
5756 switch (OpInfo.OperandType) {
5758 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
5759 ErrInfo = "Illegal immediate value for operand.";
5760 return false;
5761 }
5762 break;
5776 break;
5790 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
5791 ErrInfo = "Illegal immediate value for operand.";
5792 return false;
5793 }
5794 break;
5795 }
5800 if (ST.has64BitLiterals() && Desc.getSize() != 4 && MO.isImm() &&
5801 !isInlineConstant(MI, i) &&
5803 OpInfo.OperandType ==
5805 ErrInfo = "illegal 64-bit immediate value for operand.";
5806 return false;
5807 }
5808 break;
5811 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, i)) {
5812 ErrInfo = "Expected inline constant for operand.";
5813 return false;
5814 }
5815 break;
5818 break;
5823 // Check if this operand is an immediate.
5824 // FrameIndex operands will be replaced by immediates, so they are
5825 // allowed.
5826 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
5827 ErrInfo = "Expected immediate, but got non-immediate";
5828 return false;
5829 }
5830 break;
5834 break;
5835 default:
5836 if (OpInfo.isGenericType())
5837 continue;
5838 break;
5839 }
5840 }
5841
5842 // Verify SDWA
5843 if (isSDWA(MI)) {
5844 if (!ST.hasSDWA()) {
5845 ErrInfo = "SDWA is not supported on this target";
5846 return false;
5847 }
5848
5849 for (auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,
5850 AMDGPU::OpName::dst_sel}) {
5851 const MachineOperand *MO = getNamedOperand(MI, Op);
5852 if (!MO)
5853 continue;
5854 int64_t Imm = MO->getImm();
5856 ErrInfo = "Invalid SDWA selection";
5857 return false;
5858 }
5859 }
5860
5861 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
5862
5863 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
5864 if (OpIdx == -1)
5865 continue;
5866 const MachineOperand &MO = MI.getOperand(OpIdx);
5867
5868 if (!ST.hasSDWAScalar()) {
5869 // Only VGPRS on VI
5870 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
5871 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
5872 return false;
5873 }
5874 } else {
5875 // No immediates on GFX9
5876 if (!MO.isReg()) {
5877 ErrInfo =
5878 "Only reg allowed as operands in SDWA instructions on GFX9+";
5879 return false;
5880 }
5881 }
5882 }
5883
5884 if (!ST.hasSDWAOmod()) {
5885 // No omod allowed on VI
5886 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
5887 if (OMod != nullptr &&
5888 (!OMod->isImm() || OMod->getImm() != 0)) {
5889 ErrInfo = "OMod not allowed in SDWA instructions on VI";
5890 return false;
5891 }
5892 }
5893
5894 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
5895 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
5896 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
5897 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
5898 const MachineOperand *Src0ModsMO =
5899 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
5900 unsigned Mods = Src0ModsMO->getImm();
5901 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||
5902 Mods & SISrcMods::SEXT) {
5903 ErrInfo = "sext, abs and neg are not allowed on this instruction";
5904 return false;
5905 }
5906 }
5907
5908 uint32_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
5909 if (isVOPC(BasicOpcode)) {
5910 if (!ST.hasSDWASdst() && DstIdx != -1) {
5911 // Only vcc allowed as dst on VI for VOPC
5912 const MachineOperand &Dst = MI.getOperand(DstIdx);
5913 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
5914 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
5915 return false;
5916 }
5917 } else if (!ST.hasSDWAOutModsVOPC()) {
5918 // No clamp allowed on GFX9 for VOPC
5919 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
5920 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
5921 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
5922 return false;
5923 }
5924
5925 // No omod allowed on GFX9 for VOPC
5926 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
5927 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
5928 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
5929 return false;
5930 }
5931 }
5932 }
5933
5934 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
5935 if (DstUnused && DstUnused->isImm() &&
5936 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
5937 const MachineOperand &Dst = MI.getOperand(DstIdx);
5938 if (!Dst.isReg() || !Dst.isTied()) {
5939 ErrInfo = "Dst register should have tied register";
5940 return false;
5941 }
5942
5943 const MachineOperand &TiedMO =
5944 MI.getOperand(MI.findTiedOperandIdx(DstIdx));
5945 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
5946 ErrInfo =
5947 "Dst register should be tied to implicit use of preserved register";
5948 return false;
5949 }
5950 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {
5951 ErrInfo = "Dst register should use same physical register as preserved";
5952 return false;
5953 }
5954 }
5955 }
5956
5957 if (isDPP(MI) && !ST.hasDPPSrc1SGPR() && Src1Idx != -1) {
5958 const MachineOperand &Src1MO = MI.getOperand(Src1Idx);
5959 if (Src1MO.isReg() && RI.isSGPRReg(MRI, Src1MO.getReg())) {
5960 ErrInfo = "DPP src1 cannot be SGPR on this subtarget";
5961 return false;
5962 }
5963 if (Src1MO.isImm()) {
5964 ErrInfo = "DPP src1 cannot be an immediate on this subtarget";
5965 return false;
5966 }
5967 }
5968
5969 // Verify MIMG / VIMAGE / VSAMPLE
5970 if (isImage(Opcode) && !MI.mayStore()) {
5971 // Ensure that the return type used is large enough for all the options
5972 // being used TFE/LWE require an extra result register.
5973 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
5974 if (DMask) {
5975 uint64_t DMaskImm = DMask->getImm();
5976 uint32_t RegCount = isGather4(Opcode) ? 4 : llvm::popcount(DMaskImm);
5977 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
5978 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
5979 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
5980
5981 // Adjust for packed 16 bit values
5982 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
5983 RegCount = divideCeil(RegCount, 2);
5984
5985 // Adjust if using LWE or TFE
5986 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
5987 RegCount += 1;
5988
5989 const uint32_t DstIdx =
5990 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
5991 const MachineOperand &Dst = MI.getOperand(DstIdx);
5992 if (Dst.isReg()) {
5993 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
5994 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
5995 if (RegCount > DstSize) {
5996 ErrInfo = "Image instruction returns too many registers for dst "
5997 "register class";
5998 return false;
5999 }
6000 }
6001 }
6002 }
6003
6004 // Verify VOP*. Ignore multiple sgpr operands on writelane.
6005 if (isVALU(MI, /*AllowLDSDMA=*/false) &&
6006 Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
6007 unsigned ConstantBusCount = 0;
6008 bool UsesLiteral = false;
6009 const MachineOperand *LiteralVal = nullptr;
6010
6011 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);
6012 if (ImmIdx != -1) {
6013 ++ConstantBusCount;
6014 UsesLiteral = true;
6015 LiteralVal = &MI.getOperand(ImmIdx);
6016 }
6017
6018 SmallVector<Register, 2> SGPRsUsed;
6019 Register SGPRUsed;
6020
6021 // Only look at the true operands. Only a real operand can use the constant
6022 // bus, and we don't want to check pseudo-operands like the source modifier
6023 // flags.
6024 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
6025 if (OpIdx == -1)
6026 continue;
6027 const MachineOperand &MO = MI.getOperand(OpIdx);
6028 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) {
6029 if (MO.isReg()) {
6030 SGPRUsed = MO.getReg();
6031 if (!llvm::is_contained(SGPRsUsed, SGPRUsed)) {
6032 ++ConstantBusCount;
6033 SGPRsUsed.push_back(SGPRUsed);
6034 }
6035 } else if (!MO.isFI()) { // Treat FI like a register.
6036 if (!UsesLiteral) {
6037 ++ConstantBusCount;
6038 UsesLiteral = true;
6039 LiteralVal = &MO;
6040 } else if (!MO.isIdenticalTo(*LiteralVal)) {
6041 assert(isVOP2(MI) || isVOP3(MI));
6042 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
6043 return false;
6044 }
6045 }
6046 }
6047 }
6048
6049 SGPRUsed = findImplicitSGPRRead(MI);
6050 if (SGPRUsed) {
6051 // Implicit uses may safely overlap true operands
6052 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
6053 return !RI.regsOverlap(SGPRUsed, SGPR);
6054 })) {
6055 ++ConstantBusCount;
6056 SGPRsUsed.push_back(SGPRUsed);
6057 }
6058 }
6059
6060 // v_writelane_b32 is an exception from constant bus restriction:
6061 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
6062 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
6063 Opcode != AMDGPU::V_WRITELANE_B32) {
6064 ErrInfo = "VOP* instruction violates constant bus restriction";
6065 return false;
6066 }
6067
6068 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
6069 ErrInfo = "VOP3 instruction uses literal";
6070 return false;
6071 }
6072 }
6073
6074 // Special case for writelane - this can break the multiple constant bus rule,
6075 // but still can't use more than one SGPR register
6076 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
6077 unsigned SGPRCount = 0;
6078 Register SGPRUsed;
6079
6080 for (int OpIdx : {Src0Idx, Src1Idx}) {
6081 if (OpIdx == -1)
6082 break;
6083
6084 const MachineOperand &MO = MI.getOperand(OpIdx);
6085
6086 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) {
6087 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
6088 if (MO.getReg() != SGPRUsed)
6089 ++SGPRCount;
6090 SGPRUsed = MO.getReg();
6091 }
6092 }
6093 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
6094 ErrInfo = "WRITELANE instruction violates constant bus restriction";
6095 return false;
6096 }
6097 }
6098 }
6099
6100 if (Opcode == AMDGPU::ASYNCMARK || Opcode == AMDGPU::WAIT_ASYNCMARK) {
6101 int64_t Mask = MI.getOperand(MI.getNumExplicitOperands() - 1).getImm();
6102 if (!AMDGPU::isValidAsyncStageMask(Mask)) {
6103 ErrInfo = "invalid asyncmark stage mask";
6104 return false;
6105 }
6106 }
6107
6108 // An offset past the addressable VGPRs is an out-of-bounds access, which is
6109 // allowed, but each dword must still be a VGPR.
6110 if (const auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
6111 const MachineOperand &Offset = LdSt->getOffsetOp();
6112 if (!Offset.isImm() || Offset.getImm() < 0 ||
6113 Offset.getImm() + LdSt->getBitWidth() / 32 >
6114 AMDGPU::VGPR_32RegClass.getNumRegs()) {
6115 ErrInfo = "invalid VGPR address space offset";
6116 return false;
6117 }
6118 }
6119
6120 // Verify misc. restrictions on specific instructions.
6121 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
6122 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
6123 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
6124 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
6125 const MachineOperand &Src2 = MI.getOperand(Src2Idx);
6126 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
6127 if (!compareMachineOp(Src0, Src1) &&
6128 !compareMachineOp(Src0, Src2)) {
6129 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
6130 return false;
6131 }
6132 }
6133 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &
6134 SISrcMods::ABS) ||
6135 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() &
6136 SISrcMods::ABS) ||
6137 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() &
6138 SISrcMods::ABS)) {
6139 ErrInfo = "ABS not allowed in VOP3B instructions";
6140 return false;
6141 }
6142 }
6143
6144 if (isSOP2(MI) || isSOPC(MI)) {
6145 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
6146 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
6147
6148 if (!isRegOrFI(Src0) && !isRegOrFI(Src1) &&
6149 !isInlineConstant(Src0, Desc.operands()[Src0Idx]) &&
6150 !isInlineConstant(Src1, Desc.operands()[Src1Idx]) &&
6151 !Src0.isIdenticalTo(Src1)) {
6152 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
6153 return false;
6154 }
6155 }
6156
6157 if (isSOPK(MI)) {
6158 const auto *Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
6159 if (Desc.isBranch()) {
6160 if (!Op->isMBB()) {
6161 ErrInfo = "invalid branch target for SOPK instruction";
6162 return false;
6163 }
6164 } else {
6165 uint64_t Imm = Op->getImm();
6166 if (sopkIsZext(Opcode)) {
6167 if (!isUInt<16>(Imm)) {
6168 ErrInfo = "invalid immediate for SOPK instruction";
6169 return false;
6170 }
6171 } else {
6172 if (!isInt<16>(Imm)) {
6173 ErrInfo = "invalid immediate for SOPK instruction";
6174 return false;
6175 }
6176 }
6177 }
6178 }
6179
6180 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
6181 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
6182 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
6183 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
6184 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
6185 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
6186
6187 const unsigned StaticNumOps =
6188 Desc.getNumOperands() + Desc.implicit_uses().size();
6189 const unsigned NumImplicitOps = IsDst ? 2 : 1;
6190
6191 // Require additional implicit operands. This allows a fixup done by the
6192 // post RA scheduler where the main implicit operand is killed and
6193 // implicit-defs are added for sub-registers that remain live after this
6194 // instruction.
6195 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
6196 ErrInfo = "missing implicit register operands";
6197 return false;
6198 }
6199
6200 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
6201 if (IsDst) {
6202 if (!Dst->isUse()) {
6203 ErrInfo = "v_movreld_b32 vdst should be a use operand";
6204 return false;
6205 }
6206
6207 unsigned UseOpIdx;
6208 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
6209 UseOpIdx != StaticNumOps + 1) {
6210 ErrInfo = "movrel implicit operands should be tied";
6211 return false;
6212 }
6213 }
6214
6215 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
6216 const MachineOperand &ImpUse
6217 = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
6218 if (!ImpUse.isReg() || !ImpUse.isUse() ||
6219 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
6220 ErrInfo = "src0 should be subreg of implicit vector use";
6221 return false;
6222 }
6223 }
6224
6225 // Make sure we aren't losing exec uses in the td files. This mostly requires
6226 // being careful when using let Uses to try to add other use registers.
6227 if (shouldReadExec(MI)) {
6228 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
6229 ErrInfo = "VALU instruction does not implicitly read exec mask";
6230 return false;
6231 }
6232 }
6233
6234 if (isSMRD(MI)) {
6235 if (MI.mayStore() &&
6236 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
6237 // The register offset form of scalar stores may only use m0 as the
6238 // soffset register.
6239 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset);
6240 if (Soff && Soff->getReg() != AMDGPU::M0) {
6241 ErrInfo = "scalar stores must use m0 as offset register";
6242 return false;
6243 }
6244 }
6245 }
6246
6247 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
6248 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
6249 if (Offset->getImm() != 0) {
6250 ErrInfo = "subtarget does not support offsets in flat instructions";
6251 return false;
6252 }
6253 }
6254
6255 if (isDS(MI) && !ST.hasGDS()) {
6256 const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds);
6257 if (GDSOp && GDSOp->getImm() != 0) {
6258 ErrInfo = "GDS is not supported on this subtarget";
6259 return false;
6260 }
6261 }
6262
6263 if (isImage(MI)) {
6264 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
6265 if (DimOp) {
6266 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
6267 AMDGPU::OpName::vaddr0);
6268 AMDGPU::OpName RSrcOpName =
6269 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
6270 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, RSrcOpName);
6271 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
6272 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
6273 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
6274 const AMDGPU::MIMGDimInfo *Dim =
6276
6277 if (!Dim) {
6278 ErrInfo = "dim is out of range";
6279 return false;
6280 }
6281
6282 bool IsA16 = false;
6283 if (ST.hasR128A16()) {
6284 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128);
6285 IsA16 = R128A16->getImm() != 0;
6286 } else if (ST.hasA16()) {
6287 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16);
6288 IsA16 = A16->getImm() != 0;
6289 }
6290
6291 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
6292
6293 unsigned AddrWords =
6294 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16());
6295
6296 unsigned VAddrWords;
6297 if (IsNSA) {
6298 VAddrWords = RsrcIdx - VAddr0Idx;
6299 if (ST.hasPartialNSAEncoding() &&
6300 AddrWords > ST.getNSAMaxSize(isVSAMPLE(MI))) {
6301 unsigned LastVAddrIdx = RsrcIdx - 1;
6302 VAddrWords += getOpSize(MI, LastVAddrIdx) / 4 - 1;
6303 }
6304 } else {
6305 VAddrWords = getOpSize(MI, VAddr0Idx) / 4;
6306 if (AddrWords > 12)
6307 AddrWords = 16;
6308 }
6309
6310 if (VAddrWords != AddrWords) {
6311 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
6312 << " but got " << VAddrWords << "\n");
6313 ErrInfo = "bad vaddr size";
6314 return false;
6315 }
6316 }
6317 }
6318
6319 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
6320 if (DppCt) {
6321 using namespace AMDGPU::DPP;
6322
6323 unsigned DC = DppCt->getImm();
6324 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
6325 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
6326 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
6327 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
6328 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
6329 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
6330 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
6331 ErrInfo = "Invalid dpp_ctrl value";
6332 return false;
6333 }
6334 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
6335 !ST.hasDPPWavefrontShifts()) {
6336 ErrInfo = "Invalid dpp_ctrl value: "
6337 "wavefront shifts are not supported on GFX10+";
6338 return false;
6339 }
6340 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
6341 !ST.hasDPPBroadcasts()) {
6342 ErrInfo = "Invalid dpp_ctrl value: "
6343 "broadcasts are not supported on GFX10+";
6344 return false;
6345 }
6346 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
6347 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
6348 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
6349 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
6350 !ST.hasGFX90AInsts()) {
6351 ErrInfo = "Invalid dpp_ctrl value: "
6352 "row_newbroadcast/row_share is not supported before "
6353 "GFX90A/GFX10";
6354 return false;
6355 }
6356 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
6357 ErrInfo = "Invalid dpp_ctrl value: "
6358 "row_share and row_xmask are not supported before GFX10";
6359 return false;
6360 }
6361 }
6362
6363 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
6365 ST.hasFeature(AMDGPU::FeatureDPALU_DPP) &&
6366 AMDGPU::isDPALU_DPP(Desc, *this, ST)) {
6367 ErrInfo = "Invalid dpp_ctrl value: "
6368 "DP ALU dpp only support row_newbcast";
6369 return false;
6370 }
6371 }
6372
6373 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
6374 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
6375 AMDGPU::OpName DataName =
6376 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;
6377 const MachineOperand *Data = getNamedOperand(MI, DataName);
6378 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1);
6379 if (Data && !Data->isReg())
6380 Data = nullptr;
6381
6382 if (!ST.hasGFX90AInsts()) {
6383 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) ||
6384 (Data && RI.isAGPR(MRI, Data->getReg())) ||
6385 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) {
6386 ErrInfo = "Invalid register class: "
6387 "agpr loads and stores not supported on this GPU";
6388 return false;
6389 }
6390 }
6391 }
6392
6393 if (ST.needsAlignedVGPRs()) {
6394 const auto isAlignedReg = [&MI, &MRI, this](AMDGPU::OpName OpName) -> bool {
6396 if (!Op)
6397 return true;
6398 Register Reg = Op->getReg();
6399 if (Reg.isPhysical())
6400 return !(RI.getHWRegIndex(Reg) & 1);
6401 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
6402 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
6403 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);
6404 };
6405
6406 if (isMIMG(MI)) {
6407 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
6408 ErrInfo = "Subtarget requires even aligned vector registers "
6409 "for vaddr operand of image instructions";
6410 return false;
6411 }
6412 }
6413 }
6414
6415 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {
6416 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0);
6417 if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) {
6418 ErrInfo = "Invalid register class: "
6419 "v_accvgpr_write with an SGPR is not supported on this GPU";
6420 return false;
6421 }
6422 }
6423
6424 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
6425 const MachineOperand &SrcOp = MI.getOperand(1);
6426 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
6427 ErrInfo = "pseudo expects only physical SGPRs";
6428 return false;
6429 }
6430 }
6431
6432 if (const MachineOperand *CPol = getNamedOperand(MI, AMDGPU::OpName::cpol)) {
6433 if (CPol->getImm() & AMDGPU::CPol::SCAL) {
6434 if (!ST.hasScaleOffset()) {
6435 ErrInfo = "subtarget does not support offset scaling";
6436 return false;
6437 }
6438 if (!AMDGPU::supportsScaleOffset(*this, MI.getOpcode())) {
6439 ErrInfo = "instruction does not support offset scaling";
6440 return false;
6441 }
6442 }
6443 }
6444
6445 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6446 if (AMDGPU::isSingleSGPRReadInst(Opcode)) {
6447 for (unsigned I = 0; I < 3; ++I) {
6449 return false;
6450 }
6451 }
6452
6453 if (ST.hasFlatScratchHiInB64InstHazard() && isSALU(MI) &&
6454 MI.readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, nullptr)) {
6455 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::sdst);
6456 if ((Dst && RI.getRegClassForReg(MRI, Dst->getReg()) ==
6457 &AMDGPU::SReg_64RegClass) ||
6458 Opcode == AMDGPU::S_BITCMP0_B64 || Opcode == AMDGPU::S_BITCMP1_B64) {
6459 ErrInfo = "Instruction cannot read flat_scratch_base_hi";
6460 return false;
6461 }
6462 }
6463
6464 // GFX950 v_pk_{add,mul,fma}_f32 must not have an invalid op_sel.
6465 if (!isLegalPkF32OpSel(*this, MI)) {
6466 ErrInfo = "GFX950 v_pk_{add,mul,fma}_f32 has an invalid op_sel.";
6467 return false;
6468 }
6469
6470 return true;
6471}
6472
6474 if (MI.getOpcode() == AMDGPU::S_MOV_B32) {
6475 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6476 return MI.getOperand(1).isReg() || RI.isAGPR(MRI, MI.getOperand(0).getReg())
6477 ? AMDGPU::COPY
6478 : AMDGPU::V_MOV_B32_e32;
6479 }
6480 return getVALUOp(MI.getOpcode());
6481}
6482
6483// It is more readable to list mapped opcodes on the same line.
6484// clang-format off
6485
6486unsigned SIInstrInfo::getVALUOp(unsigned Opc) const {
6487 switch (Opc) {
6488 default: return AMDGPU::INSTRUCTION_LIST_END;
6489 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
6490 case AMDGPU::COPY: return AMDGPU::COPY;
6491 case AMDGPU::PHI: return AMDGPU::PHI;
6492 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
6493 case AMDGPU::WQM: return AMDGPU::WQM;
6494 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
6495 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
6496 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
6497 case AMDGPU::S_ADD_I32:
6498 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
6499 case AMDGPU::S_ADDC_U32:
6500 return AMDGPU::V_ADDC_U32_e32;
6501 case AMDGPU::S_SUB_I32:
6502 return ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
6503 // FIXME: These are not consistently handled, and selected when the carry is
6504 // used.
6505 case AMDGPU::S_ADD_U32:
6506 return AMDGPU::V_ADD_CO_U32_e32;
6507 case AMDGPU::S_SUB_U32:
6508 return AMDGPU::V_SUB_CO_U32_e32;
6509 case AMDGPU::S_ADD_U64_PSEUDO:
6510 return AMDGPU::V_ADD_U64_PSEUDO;
6511 case AMDGPU::S_SUB_U64_PSEUDO:
6512 return AMDGPU::V_SUB_U64_PSEUDO;
6513 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
6514 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
6515 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
6516 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
6517 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
6518 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
6519 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
6520 case AMDGPU::S_XNOR_B32:
6521 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
6522 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
6523 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
6524 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
6525 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
6526 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
6527 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
6528 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
6529 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
6530 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
6531 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
6532 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
6533 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
6534 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
6535 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
6536 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
6537 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
6538 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
6539 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
6540 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
6541 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
6542 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
6543 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
6544 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
6545 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
6546 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
6547 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
6548 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
6549 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
6550 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
6551 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
6552 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
6553 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
6554 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
6555 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
6556 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
6557 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
6558 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
6559 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
6560 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
6561 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
6562 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
6563 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
6564 case AMDGPU::S_CVT_F32_F16:
6565 case AMDGPU::S_CVT_HI_F32_F16:
6566 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
6567 : AMDGPU::V_CVT_F32_F16_fake16_e64;
6568 case AMDGPU::S_CVT_F16_F32:
6569 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
6570 : AMDGPU::V_CVT_F16_F32_fake16_e64;
6571 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
6572 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
6573 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
6574 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
6575 case AMDGPU::S_CEIL_F16:
6576 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
6577 : AMDGPU::V_CEIL_F16_fake16_e64;
6578 case AMDGPU::S_FLOOR_F16:
6579 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
6580 : AMDGPU::V_FLOOR_F16_fake16_e64;
6581 case AMDGPU::S_TRUNC_F16:
6582 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e64
6583 : AMDGPU::V_TRUNC_F16_fake16_e64;
6584 case AMDGPU::S_RNDNE_F16:
6585 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e64
6586 : AMDGPU::V_RNDNE_F16_fake16_e64;
6587 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
6588 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
6589 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
6590 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
6591 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
6592 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
6593 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
6594 case AMDGPU::S_ADD_F16:
6595 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e64
6596 : AMDGPU::V_ADD_F16_fake16_e64;
6597 case AMDGPU::S_SUB_F16:
6598 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e64
6599 : AMDGPU::V_SUB_F16_fake16_e64;
6600 case AMDGPU::S_MIN_F16:
6601 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e64
6602 : AMDGPU::V_MIN_F16_fake16_e64;
6603 case AMDGPU::S_MAX_F16:
6604 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e64
6605 : AMDGPU::V_MAX_F16_fake16_e64;
6606 case AMDGPU::S_MINIMUM_F16:
6607 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
6608 : AMDGPU::V_MINIMUM_F16_fake16_e64;
6609 case AMDGPU::S_MAXIMUM_F16:
6610 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
6611 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
6612 case AMDGPU::S_MUL_F16:
6613 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e64
6614 : AMDGPU::V_MUL_F16_fake16_e64;
6615 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
6616 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
6617 case AMDGPU::S_FMAC_F16:
6618 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e64
6619 : AMDGPU::V_FMAC_F16_fake16_e64;
6620 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
6621 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
6622 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
6623 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
6624 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
6625 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
6626 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
6627 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
6628 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
6629 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
6630 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
6631 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
6632 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
6633 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
6634 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
6635 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
6636 case AMDGPU::S_CMP_LT_F16:
6637 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
6638 : AMDGPU::V_CMP_LT_F16_fake16_e64;
6639 case AMDGPU::S_CMP_EQ_F16:
6640 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
6641 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
6642 case AMDGPU::S_CMP_LE_F16:
6643 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
6644 : AMDGPU::V_CMP_LE_F16_fake16_e64;
6645 case AMDGPU::S_CMP_GT_F16:
6646 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
6647 : AMDGPU::V_CMP_GT_F16_fake16_e64;
6648 case AMDGPU::S_CMP_LG_F16:
6649 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
6650 : AMDGPU::V_CMP_LG_F16_fake16_e64;
6651 case AMDGPU::S_CMP_GE_F16:
6652 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
6653 : AMDGPU::V_CMP_GE_F16_fake16_e64;
6654 case AMDGPU::S_CMP_O_F16:
6655 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
6656 : AMDGPU::V_CMP_O_F16_fake16_e64;
6657 case AMDGPU::S_CMP_U_F16:
6658 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
6659 : AMDGPU::V_CMP_U_F16_fake16_e64;
6660 case AMDGPU::S_CMP_NGE_F16:
6661 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
6662 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
6663 case AMDGPU::S_CMP_NLG_F16:
6664 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
6665 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
6666 case AMDGPU::S_CMP_NGT_F16:
6667 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
6668 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
6669 case AMDGPU::S_CMP_NLE_F16:
6670 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
6671 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
6672 case AMDGPU::S_CMP_NEQ_F16:
6673 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
6674 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
6675 case AMDGPU::S_CMP_NLT_F16:
6676 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
6677 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
6678 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
6679 case AMDGPU::V_S_EXP_F16_e64:
6680 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e64
6681 : AMDGPU::V_EXP_F16_fake16_e64;
6682 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
6683 case AMDGPU::V_S_LOG_F16_e64:
6684 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e64
6685 : AMDGPU::V_LOG_F16_fake16_e64;
6686 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
6687 case AMDGPU::V_S_RCP_F16_e64:
6688 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e64
6689 : AMDGPU::V_RCP_F16_fake16_e64;
6690 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
6691 case AMDGPU::V_S_RSQ_F16_e64:
6692 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e64
6693 : AMDGPU::V_RSQ_F16_fake16_e64;
6694 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
6695 case AMDGPU::V_S_SQRT_F16_e64:
6696 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e64
6697 : AMDGPU::V_SQRT_F16_fake16_e64;
6698 }
6700 "Unexpected scalar opcode without corresponding vector one!");
6701}
6702
6703// clang-format on
6704
6708 const DebugLoc &DL, Register Reg,
6709 bool IsSCCLive,
6710 SlotIndexes *Indexes) const {
6711 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6712 const SIInstrInfo *TII = ST.getInstrInfo();
6714 if (IsSCCLive) {
6715 // Insert two move instructions, one to save the original value of EXEC and
6716 // the other to turn on all bits in EXEC. This is required as we can't use
6717 // the single instruction S_OR_SAVEEXEC that clobbers SCC.
6718 auto StoreExecMI = BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), Reg)
6720 auto FlipExecMI =
6721 BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), LMC.ExecReg).addImm(-1);
6722 if (Indexes) {
6723 Indexes->insertMachineInstrInMaps(*StoreExecMI);
6724 Indexes->insertMachineInstrInMaps(*FlipExecMI);
6725 }
6726 } else {
6727 auto SaveExec =
6728 BuildMI(MBB, MBBI, DL, TII->get(LMC.OrSaveExecOpc), Reg).addImm(-1);
6729 SaveExec->getOperand(3).setIsDead(); // Mark SCC as dead.
6730 if (Indexes)
6731 Indexes->insertMachineInstrInMaps(*SaveExec);
6732 }
6733}
6734
6737 const DebugLoc &DL, Register Reg,
6738 SlotIndexes *Indexes) const {
6740 auto ExecRestoreMI = BuildMI(MBB, MBBI, DL, get(LMC.MovOpc), LMC.ExecReg)
6741 .addReg(Reg, RegState::Kill);
6742 if (Indexes)
6743 Indexes->insertMachineInstrInMaps(*ExecRestoreMI);
6744}
6745
6749 "Not a whole wave func");
6750 MachineBasicBlock &MBB = *MF.begin();
6751 for (MachineInstr &MI : MBB)
6752 if (MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||
6753 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
6754 return &MI;
6755
6756 llvm_unreachable("Couldn't find SI_SETUP_WHOLE_WAVE_FUNC instruction");
6757}
6758
6760 unsigned OpNo) const {
6761 const MCInstrDesc &Desc = get(MI.getOpcode());
6762 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
6763 Desc.operands()[OpNo].RegClass == -1) {
6764 Register Reg = MI.getOperand(OpNo).getReg();
6765
6766 if (Reg.isVirtual()) {
6767 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6768 return MRI.getRegClass(Reg);
6769 }
6770 return RI.getPhysRegBaseClass(Reg);
6771 }
6772
6773 int16_t RegClass = getOpRegClassID(Desc.operands()[OpNo]);
6774 return RegClass < 0 ? nullptr : RI.getRegClass(RegClass);
6775}
6776
6777// Convert VOP3 operand index to source number.
6778static unsigned VOP3OpIdxToSrcN(const MachineInstr &MI, unsigned OpIdx) {
6779 constexpr AMDGPU::OpName OpNames[] = {
6780 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
6781
6782 for (auto [I, OpName] : enumerate(OpNames)) {
6783 int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[I]);
6784 if (static_cast<unsigned>(SrcIdx) == OpIdx)
6785 return I;
6786 }
6787
6788 return UINT_MAX;
6789}
6790
6793 MachineBasicBlock *MBB = MI.getParent();
6794 MachineOperand &MO = MI.getOperand(OpIdx);
6795 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6796 unsigned RCID = getOpRegClassID(get(MI.getOpcode()).operands()[OpIdx]);
6797 const TargetRegisterClass *RC = RI.getRegClass(RCID);
6798 unsigned Size = RI.getRegSizeInBits(*RC);
6799 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
6800 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
6801 : AMDGPU::V_MOV_B32_e32;
6802 if (MO.isReg())
6803 Opcode = AMDGPU::COPY;
6804 else if (RI.isSGPRClass(RC))
6805 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
6806
6807 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
6808 Register Reg = MRI.createVirtualRegister(VRC);
6809 DebugLoc DL = MBB->findDebugLoc(I);
6810
6811 if (Size == 128 && AMDGPU::isPackedSingleSGPR64BitInst(MI.getOpcode()) &&
6813 // Special case for V_PK_*64 instructions: these do not have OPSEL but SGPR
6814 // sources behave like OPSEL is set replicating low 64-bits into high. VGPR
6815 // sources in turn read actual 4 registers. To move operand from an SGPR to
6816 // a VGPR we need to replicate low half.
6817 // We also do not select immediates for these instructions so it always has
6818 // to be an SGPR register here.
6819 // Operands which are not legal as per isLegalSingleSGPRReadInstOperand()
6820 // sent here specifically to fix a non-splat SGPR and shall perform a full
6821 // copy.
6822
6823 const TargetRegisterClass *VRC64 = RI.getVGPRClassForBitWidth(64);
6824 Register Low64 = MRI.createVirtualRegister(VRC64);
6825 assert(MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()));
6826 BuildMI(*MBB, I, DL, get(TargetOpcode::COPY), Low64)
6827 .addReg(MO.getReg(), {}, AMDGPU::sub0_sub1);
6828 BuildMI(*MBB, I, DL, get(TargetOpcode::REG_SEQUENCE), Reg)
6829 .addReg(Low64)
6830 .addImm(AMDGPU::sub0_sub1)
6831 .addReg(Low64)
6832 .addImm(AMDGPU::sub2_sub3);
6833 } else if (Opcode == AMDGPU::V_MOV_B16_t16_e64) {
6834 BuildMI(*MBB, I, DL, get(Opcode), Reg)
6835 .addImm(0) // src0_modifiers
6836 .add(MO)
6837 .addImm(0); // op_sel
6838 } else {
6839 BuildMI(*MBB, I, DL, get(Opcode), Reg).add(MO);
6840 }
6841
6842 MO.ChangeToRegister(Reg, false);
6843}
6844
6847 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
6848 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6849 if (!SuperReg.getReg().isVirtual())
6850 return RI.getSubReg(SuperReg.getReg(), SubIdx);
6851
6852 MachineBasicBlock *MBB = MI->getParent();
6853 const DebugLoc &DL = MI->getDebugLoc();
6854 Register SubReg = MRI.createVirtualRegister(SubRC);
6855
6856 unsigned NewSubIdx = RI.composeSubRegIndices(SuperReg.getSubReg(), SubIdx);
6857 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
6858 .addReg(SuperReg.getReg(), {}, NewSubIdx);
6859 return SubReg;
6860}
6861
6864 const MachineOperand &Op, const TargetRegisterClass *SuperRC,
6865 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6866 if (Op.isImm()) {
6867 if (SubIdx == AMDGPU::sub0)
6868 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
6869 if (SubIdx == AMDGPU::sub1)
6870 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
6871
6872 llvm_unreachable("Unhandled register index for immediate");
6873 }
6874
6875 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
6876 SubIdx, SubRC);
6877 return MachineOperand::CreateReg(SubReg, false);
6878}
6879
6880// Change the order of operands from (0, 1, 2) to (0, 2, 1)
6881void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
6882 assert(Inst.getNumExplicitOperands() == 3);
6883 MachineOperand Op1 = Inst.getOperand(1);
6884 Inst.removeOperand(1);
6885 Inst.addOperand(Op1);
6886}
6887
6889 const MCOperandInfo &OpInfo,
6890 const MachineOperand &MO) const {
6891 if (!MO.isReg())
6892 return false;
6893
6894 Register Reg = MO.getReg();
6895
6896 const TargetRegisterClass *DRC = RI.getRegClass(getOpRegClassID(OpInfo));
6897 if (Reg.isPhysical())
6898 return DRC->contains(Reg);
6899
6900 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
6901
6902 if (MO.getSubReg()) {
6903 const TargetRegisterClass *SuperRC =
6904 RI.getLargestLegalSuperClass(RC, MRI.getMF());
6905 if (!SuperRC)
6906 return false;
6907 return RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()) != nullptr;
6908 }
6909
6910 return RI.getCommonSubClass(DRC, RC) != nullptr;
6911}
6912
6914 const MachineOperand &MO) const {
6915 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6916 const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
6917 unsigned Opc = MI.getOpcode();
6918
6919 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6920 if (MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()) &&
6921 AMDGPU::isSingleSGPRReadInst(MI.getOpcode()) &&
6923 &MO))
6924 return false;
6925
6926 if (!isLegalRegOperand(MRI, OpInfo, MO))
6927 return false;
6928
6929 // check Accumulate GPR operand
6930 bool IsAGPR = RI.isAGPR(MRI, MO.getReg());
6931 if (IsAGPR && !ST.hasMAIInsts())
6932 return false;
6933 if (IsAGPR && (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
6934 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc)))
6935 return false;
6936 // Atomics should have both vdst and vdata either vgpr or agpr.
6937 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
6938 const int DataIdx = AMDGPU::getNamedOperandIdx(
6939 Opc, isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
6940 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
6941 MI.getOperand(DataIdx).isReg() &&
6942 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR)
6943 return false;
6944 if ((int)OpIdx == DataIdx) {
6945 if (VDstIdx != -1 &&
6946 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR)
6947 return false;
6948 // DS instructions with 2 src operands also must have tied RC.
6949 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
6950 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() &&
6951 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR)
6952 return false;
6953 }
6954
6955 // Check V_ACCVGPR_WRITE_B32_e64
6956 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
6957 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) &&
6958 RI.isSGPRReg(MRI, MO.getReg()))
6959 return false;
6960
6961 if (ST.hasFlatScratchHiInB64InstHazard() &&
6962 MO.getReg() == AMDGPU::SRC_FLAT_SCRATCH_BASE_HI && isSALU(MI)) {
6963 if (const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::sdst)) {
6964 if (AMDGPU::getRegBitWidth(*RI.getRegClassForReg(MRI, Dst->getReg())) ==
6965 64)
6966 return false;
6967 }
6968 if (Opc == AMDGPU::S_BITCMP0_B64 || Opc == AMDGPU::S_BITCMP1_B64)
6969 return false;
6970 }
6971 if (!ST.hasDPPSrc1SGPR() && isDPP(MI) && RI.isSGPRReg(MRI, MO.getReg()) &&
6972 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1))
6973 return false;
6974
6975 return true;
6976}
6977
6979 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
6980 const MachineOperand *MO) const {
6981 constexpr unsigned NumOps = 3;
6982 constexpr AMDGPU::OpName OpNames[NumOps * 2] = {
6983 AMDGPU::OpName::src0, AMDGPU::OpName::src1,
6984 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
6985 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
6986
6987 assert(SrcN < NumOps);
6988
6989 if (!MO) {
6990 int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[SrcN]);
6991 if (SrcIdx == -1)
6992 return true;
6993 MO = &MI.getOperand(SrcIdx);
6994 }
6995
6996 if (!MO->isReg() || !RI.isSGPRReg(MRI, MO->getReg()))
6997 return true;
6998
6999 int ModsIdx =
7000 AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[NumOps + SrcN]);
7001 if (ModsIdx == -1)
7002 return false;
7003
7004 unsigned Mods = MI.getOperand(ModsIdx).getImm();
7005 bool OpSel = Mods & SISrcMods::OP_SEL_0;
7006 bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
7007
7008 return !OpSel && !OpSelHi;
7009}
7010
7011bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
7012 const MachineOperand *MO) const {
7013 const MachineFunction &MF = *MI.getMF();
7014 const MachineRegisterInfo &MRI = MF.getRegInfo();
7015 const MCInstrDesc &InstDesc = MI.getDesc();
7016 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
7017 int64_t RegClass = getOpRegClassID(OpInfo);
7018 const TargetRegisterClass *DefinedRC =
7019 RegClass != -1 ? RI.getRegClass(RegClass) : nullptr;
7020 if (!MO)
7021 MO = &MI.getOperand(OpIdx);
7022
7023 const bool IsInlineConst = !MO->isReg() && isInlineConstant(*MO, OpInfo);
7024
7025 if (isVALU(MI, /*AllowLDSDMA=*/false) && !IsInlineConst &&
7026 usesConstantBus(MRI, *MO, OpInfo)) {
7027 const MachineOperand *UsedLiteral = nullptr;
7028
7029 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
7030 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
7031
7032 // TODO: Be more permissive with frame indexes.
7033 if (!MO->isReg() && !isInlineConstant(*MO, OpInfo)) {
7034 if (!LiteralLimit--)
7035 return false;
7036
7037 UsedLiteral = MO;
7038 }
7039
7041 if (MO->isReg())
7042 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
7043
7044 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
7045 if (i == OpIdx)
7046 continue;
7047 const MachineOperand &Op = MI.getOperand(i);
7048 if (Op.isReg()) {
7049 if (Op.isUse()) {
7050 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
7051 if (regUsesConstantBus(Op, MRI) && SGPRsUsed.insert(SGPR).second) {
7052 if (--ConstantBusLimit <= 0)
7053 return false;
7054 }
7055 }
7056 } else if (AMDGPU::isSISrcOperand(InstDesc.operands()[i]) &&
7057 !isInlineConstant(Op, InstDesc.operands()[i])) {
7058 // The same literal may be used multiple times.
7059 if (!UsedLiteral)
7060 UsedLiteral = &Op;
7061 else if (UsedLiteral->isIdenticalTo(Op))
7062 continue;
7063
7064 if (!LiteralLimit--)
7065 return false;
7066 if (--ConstantBusLimit <= 0)
7067 return false;
7068 }
7069 }
7070 } else if (!IsInlineConst && !MO->isReg() && isSALU(MI)) {
7071 // There can be at most one literal operand, but it can be repeated.
7072 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
7073 if (i == OpIdx)
7074 continue;
7075 const MachineOperand &Op = MI.getOperand(i);
7076 if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&
7077 !isInlineConstant(Op, InstDesc.operands()[i]) &&
7078 !Op.isIdenticalTo(*MO))
7079 return false;
7080
7081 // Do not fold a non-inlineable and non-register operand into an
7082 // instruction that already has a frame index. The frame index handling
7083 // code could not handle well when a frame index co-exists with another
7084 // non-register operand, unless that operand is an inlineable immediate.
7085 if (Op.isFI())
7086 return false;
7087 }
7088 }
7089
7090 if (MO->isReg()) {
7091 if (!DefinedRC)
7092 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
7093 return isLegalRegOperand(MI, OpIdx, *MO);
7094 }
7095
7096 if (MO->isImm()) {
7097 uint64_t Imm = MO->getImm();
7098 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64 ||
7099 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP64;
7100 bool Is64BitOp = Is64BitFPOp ||
7101 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
7102 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
7103 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32 ||
7104 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT64;
7105 if (Is64BitOp &&
7106 !AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm())) {
7107 if (!AMDGPU::isValid32BitLiteral(Imm, Is64BitFPOp) &&
7108 (!ST.has64BitLiterals() || InstDesc.getSize() != 4))
7109 return false;
7110
7111 // FIXME: We can use sign extended 64-bit literals, but only for signed
7112 // operands. At the moment we do not know if an operand is signed.
7113 // Such operand will be encoded as its low 32 bits and then either
7114 // correctly sign extended or incorrectly zero extended by HW.
7115 // If 64-bit literals are supported and the literal will be encoded
7116 // as full 64 bit we still can use it.
7117 if (!Is64BitFPOp && (int32_t)Imm < 0 &&
7118 (!ST.has64BitLiterals() || AMDGPU::isValid32BitLiteral(Imm, false)))
7119 return false;
7120 }
7121 }
7122
7123 // Handle non-register types that are treated like immediates.
7124 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
7125
7126 if (!DefinedRC) {
7127 // This operand expects an immediate.
7128 return true;
7129 }
7130
7131 return isImmOperandLegal(MI, OpIdx, *MO);
7132}
7133
7135 bool IsGFX950Only = ST.hasGFX950Insts();
7136 bool IsGFX940Only = ST.hasGFX940Insts();
7137
7138 if (!IsGFX950Only && !IsGFX940Only)
7139 return false;
7140
7141 if (!isVALU(MI, /*AllowLDSDMA=*/false))
7142 return false;
7143
7144 // V_COS, V_EXP, V_RCP, etc.
7145 if (isTRANS(MI))
7146 return true;
7147
7148 // DOT2, DOT2C, DOT4, etc.
7149 if (isDOT(MI))
7150 return true;
7151
7152 // MFMA, SMFMA
7153 if (isMFMA(MI))
7154 return true;
7155
7156 unsigned Opcode = MI.getOpcode();
7157 switch (Opcode) {
7158 case AMDGPU::V_CVT_PK_BF8_F32_e64:
7159 case AMDGPU::V_CVT_PK_FP8_F32_e64:
7160 case AMDGPU::V_MQSAD_PK_U16_U8_e64:
7161 case AMDGPU::V_MQSAD_U32_U8_e64:
7162 case AMDGPU::V_PK_ADD_F16:
7163 case AMDGPU::V_PK_ADD_F32:
7164 case AMDGPU::V_PK_ADD_I16:
7165 case AMDGPU::V_PK_ADD_U16:
7166 case AMDGPU::V_PK_ASHRREV_I16:
7167 case AMDGPU::V_PK_FMA_F16:
7168 case AMDGPU::V_PK_FMA_F32:
7169 case AMDGPU::V_PK_FMAC_F16_e32:
7170 case AMDGPU::V_PK_FMAC_F16_e64:
7171 case AMDGPU::V_PK_LSHLREV_B16:
7172 case AMDGPU::V_PK_LSHRREV_B16:
7173 case AMDGPU::V_PK_MAD_I16:
7174 case AMDGPU::V_PK_MAD_U16:
7175 case AMDGPU::V_PK_MAX_F16:
7176 case AMDGPU::V_PK_MAX_I16:
7177 case AMDGPU::V_PK_MAX_U16:
7178 case AMDGPU::V_PK_MIN_F16:
7179 case AMDGPU::V_PK_MIN_I16:
7180 case AMDGPU::V_PK_MIN_U16:
7181 case AMDGPU::V_PK_MOV_B32:
7182 case AMDGPU::V_PK_MUL_F16:
7183 case AMDGPU::V_PK_MUL_F32:
7184 case AMDGPU::V_PK_MUL_LO_U16:
7185 case AMDGPU::V_PK_SUB_I16:
7186 case AMDGPU::V_PK_SUB_U16:
7187 case AMDGPU::V_QSAD_PK_U16_U8_e64:
7188 return true;
7189 default:
7190 return false;
7191 }
7192}
7193
7195 MachineInstr &MI) const {
7196 unsigned Opc = MI.getOpcode();
7197 const MCInstrDesc &InstrDesc = get(Opc);
7198
7199 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
7200 MachineOperand &Src0 = MI.getOperand(Src0Idx);
7201
7202 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
7203 MachineOperand &Src1 = MI.getOperand(Src1Idx);
7204
7205 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
7206 // we need to only have one constant bus use before GFX10.
7207 bool HasImplicitSGPR = findImplicitSGPRRead(MI);
7208 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && Src0.isReg() &&
7209 RI.isSGPRReg(MRI, Src0.getReg()))
7210 legalizeOpWithMove(MI, Src0Idx);
7211
7212 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
7213 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
7214 // src0/src1 with V_READFIRSTLANE.
7215 if (Opc == AMDGPU::V_WRITELANE_B32) {
7216 const DebugLoc &DL = MI.getDebugLoc();
7217 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
7218 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7219 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
7220 .add(Src0);
7221 Src0.ChangeToRegister(Reg, false);
7222 }
7223 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
7224 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7225 const DebugLoc &DL = MI.getDebugLoc();
7226 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
7227 .add(Src1);
7228 Src1.ChangeToRegister(Reg, false);
7229 }
7230 return;
7231 }
7232
7233 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
7234 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
7235 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
7236 if (!RI.isVGPR(MRI, MI.getOperand(Src2Idx).getReg()))
7237 legalizeOpWithMove(MI, Src2Idx);
7238 }
7239
7240 // VOP2 src0 instructions support all operand types, so we don't need to check
7241 // their legality. If src1 is already legal, we don't need to do anything.
7242 if (isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src1))
7243 return;
7244
7245 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
7246 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
7247 // select is uniform.
7248 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
7249 RI.isVGPR(MRI, Src1.getReg())) {
7250 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7251 const DebugLoc &DL = MI.getDebugLoc();
7252 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
7253 .add(Src1);
7254 Src1.ChangeToRegister(Reg, false);
7255 return;
7256 }
7257
7258 // We do not use commuteInstruction here because it is too aggressive and will
7259 // commute if it is possible. We only want to commute here if it improves
7260 // legality. This can be called a fairly large number of times so don't waste
7261 // compile time pointlessly swapping and checking legality again.
7262 if (HasImplicitSGPR || !MI.isCommutable()) {
7263 legalizeOpWithMove(MI, Src1Idx);
7264 return;
7265 }
7266
7267 // If src0 can be used as src1, commuting will make the operands legal.
7268 // Otherwise we have to give up and insert a move.
7269 //
7270 // TODO: Other immediate-like operand kinds could be commuted if there was a
7271 // MachineOperand::ChangeTo* for them.
7272 if ((!Src1.isImm() && !Src1.isReg()) ||
7273 !isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src0)) {
7274 legalizeOpWithMove(MI, Src1Idx);
7275 return;
7276 }
7277
7278 int CommutedOpc = commuteOpcode(MI);
7279 if (CommutedOpc == -1) {
7280 legalizeOpWithMove(MI, Src1Idx);
7281 return;
7282 }
7283
7284 MI.setDesc(get(CommutedOpc));
7285
7286 Register Src0Reg = Src0.getReg();
7287 unsigned Src0SubReg = Src0.getSubReg();
7288 bool Src0Kill = Src0.isKill();
7289
7290 if (Src1.isImm())
7291 Src0.ChangeToImmediate(Src1.getImm());
7292 else if (Src1.isReg()) {
7293 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
7294 Src0.setSubReg(Src1.getSubReg());
7295 } else
7296 llvm_unreachable("Should only have register or immediate operands");
7297
7298 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
7299 Src1.setSubReg(Src0SubReg);
7301}
7302
7303// Legalize VOP3 operands. All operand types are supported for any operand
7304// but only one literal constant and only starting from GFX10.
7306 MachineInstr &MI) const {
7307 unsigned Opc = MI.getOpcode();
7308
7309 int VOP3Idx[3] = {
7310 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
7311 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
7312 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
7313 };
7314
7315 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
7316 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||
7317 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
7318 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||
7319 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
7320 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
7321 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {
7322 // src1 and src2 must be scalar
7323 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
7324 const DebugLoc &DL = MI.getDebugLoc();
7325 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
7326 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7327 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
7328 .add(Src1);
7329 Src1.ChangeToRegister(Reg, false);
7330 }
7331 if (VOP3Idx[2] != -1) {
7332 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
7333 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
7334 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7335 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
7336 .add(Src2);
7337 Src2.ChangeToRegister(Reg, false);
7338 }
7339 }
7340 }
7341
7342 // Find the one SGPR operand we are allowed to use.
7343 int ConstantBusLimit = ST.getConstantBusLimit(Opc);
7344 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
7345 SmallDenseSet<unsigned> SGPRsUsed;
7346 Register SGPRReg = findUsedSGPR(MI, VOP3Idx);
7347 if (SGPRReg) {
7348 SGPRsUsed.insert(SGPRReg);
7349 --ConstantBusLimit;
7350 }
7351
7352 for (int Idx : VOP3Idx) {
7353 if (Idx == -1)
7354 break;
7355 MachineOperand &MO = MI.getOperand(Idx);
7356
7357 if (!MO.isReg()) {
7358 if (isInlineConstant(MO, get(Opc).operands()[Idx]))
7359 continue;
7360
7361 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
7362 --LiteralLimit;
7363 --ConstantBusLimit;
7364 continue;
7365 }
7366
7367 --LiteralLimit;
7368 --ConstantBusLimit;
7369 legalizeOpWithMove(MI, Idx);
7370 continue;
7371 }
7372
7373 if (!RI.isSGPRClass(RI.getRegClassForReg(MRI, MO.getReg())))
7374 continue; // VGPRs are legal
7375
7376 // We can use one SGPR in each VOP3 instruction prior to GFX10
7377 // and two starting from GFX10.
7378 if (SGPRsUsed.count(MO.getReg()))
7379 continue;
7380 if (ConstantBusLimit > 0) {
7381 SGPRsUsed.insert(MO.getReg());
7382 --ConstantBusLimit;
7383 continue;
7384 }
7385
7386 // If we make it this far, then the operand is not legal and we must
7387 // legalize it.
7388 legalizeOpWithMove(MI, Idx);
7389 }
7390
7391 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
7392 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
7393 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
7394 Opc == AMDGPU::V_FMAC_F16_t16_e64) &&
7395 !RI.isVGPR(MRI, MI.getOperand(VOP3Idx[2]).getReg()))
7396 legalizeOpWithMove(MI, VOP3Idx[2]);
7397
7398 // Fix the register class of single-sgpr-read instructions on gfx12+. See
7399 // SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
7401 for (unsigned I = 0; I < 3; ++I) {
7402 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, /*SrcN=*/I))
7403 legalizeOpWithMove(MI, VOP3Idx[I]);
7404 }
7405 }
7406}
7407
7410 const TargetRegisterClass *DstRC /*=nullptr*/) const {
7411 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
7412 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
7413 if (DstRC)
7414 SRC = RI.getCommonSubClass(SRC, DstRC);
7415
7416 Register DstReg = MRI.createVirtualRegister(SRC);
7417 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
7418
7419 if (RI.hasAGPRs(VRC)) {
7420 VRC = RI.getEquivalentVGPRClass(VRC);
7421 Register NewSrcReg = MRI.createVirtualRegister(VRC);
7422 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
7423 get(TargetOpcode::COPY), NewSrcReg)
7424 .addReg(SrcReg);
7425 SrcReg = NewSrcReg;
7426 }
7427
7428 if (SubRegs == 1) {
7429 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
7430 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
7431 .addReg(SrcReg);
7432 return DstReg;
7433 }
7434
7436 for (unsigned i = 0; i < SubRegs; ++i) {
7437 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
7438 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
7439 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
7440 .addReg(SrcReg, {}, RI.getSubRegFromChannel(i));
7441 SRegs.push_back(SGPR);
7442 }
7443
7445 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
7446 get(AMDGPU::REG_SEQUENCE), DstReg);
7447 for (unsigned i = 0; i < SubRegs; ++i) {
7448 MIB.addReg(SRegs[i]);
7449 MIB.addImm(RI.getSubRegFromChannel(i));
7450 }
7451 return DstReg;
7452}
7453
7455 MachineInstr &MI) const {
7456
7457 // If the pointer is store in VGPRs, then we need to move them to
7458 // SGPRs using v_readfirstlane. This is safe because we only select
7459 // loads with uniform pointers to SMRD instruction so we know the
7460 // pointer value is uniform.
7461 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
7462 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
7463 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
7464 SBase->setReg(SGPR);
7465 }
7466 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset);
7467 if (SOff && !RI.isSGPRReg(MRI, SOff->getReg())) {
7468 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
7469 SOff->setReg(SGPR);
7470 }
7471}
7472
7474 unsigned Opc = Inst.getOpcode();
7475 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
7476 if (OldSAddrIdx < 0)
7477 return false;
7478
7479 assert(isSegmentSpecificFLAT(Inst) || (isFLAT(Inst) && ST.hasFlatGVSMode()));
7480
7481 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
7482 if (NewOpc < 0)
7484 if (NewOpc < 0)
7485 return false;
7486
7488 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx);
7489 if (RI.isSGPRReg(MRI, SAddr.getReg()))
7490 return false;
7491
7492 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
7493 if (NewVAddrIdx < 0)
7494 return false;
7495
7496 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
7497
7498 // Check vaddr, it shall be zero or absent.
7499 MachineInstr *VAddrDef = nullptr;
7500 if (OldVAddrIdx >= 0) {
7501 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx);
7502 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg());
7503 if (!VAddrDef || !VAddrDef->isMoveImmediate() ||
7504 !VAddrDef->getOperand(1).isImm() ||
7505 VAddrDef->getOperand(1).getImm() != 0)
7506 return false;
7507 }
7508
7509 const MCInstrDesc &NewDesc = get(NewOpc);
7510 Inst.setDesc(NewDesc);
7511
7512 // Callers expect iterator to be valid after this call, so modify the
7513 // instruction in place.
7514 if (OldVAddrIdx == NewVAddrIdx) {
7515 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx);
7516 // Clear use list from the old vaddr holding a zero register.
7517 MRI.removeRegOperandFromUseList(&NewVAddr);
7518 MRI.moveOperands(&NewVAddr, &SAddr, 1);
7519 Inst.removeOperand(OldSAddrIdx);
7520 // Update the use list with the pointer we have just moved from vaddr to
7521 // saddr position. Otherwise new vaddr will be missing from the use list.
7522 MRI.removeRegOperandFromUseList(&NewVAddr);
7523 MRI.addRegOperandToUseList(&NewVAddr);
7524 } else {
7525 assert(OldSAddrIdx == NewVAddrIdx);
7526
7527 if (OldVAddrIdx >= 0) {
7528 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc,
7529 AMDGPU::OpName::vdst_in);
7530
7531 // removeOperand doesn't try to fixup tied operand indexes at it goes, so
7532 // it asserts. Untie the operands for now and retie them afterwards.
7533 if (NewVDstIn != -1) {
7534 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in);
7535 Inst.untieRegOperand(OldVDstIn);
7536 }
7537
7538 Inst.removeOperand(OldVAddrIdx);
7539
7540 if (NewVDstIn != -1) {
7541 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst);
7542 Inst.tieOperands(NewVDst, NewVDstIn);
7543 }
7544 }
7545 }
7546
7547 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg()))
7548 VAddrDef->eraseFromParent();
7549
7550 return true;
7551}
7552
7553// FIXME: Remove this when SelectionDAG is obsoleted.
7555 MachineInstr &MI) const {
7556 if (!isSegmentSpecificFLAT(MI) && !ST.hasFlatGVSMode())
7557 return;
7558
7559 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
7560 // thinks they are uniform, so a readfirstlane should be valid.
7561 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr);
7562 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg())))
7563 return;
7564
7566 return;
7567
7568 const TargetRegisterClass *DeclaredRC =
7569 getRegClass(MI.getDesc(), SAddr->getOperandNo());
7570
7571 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI, DeclaredRC);
7572 SAddr->setReg(ToSGPR);
7573}
7574
7577 const TargetRegisterClass *DstRC,
7580 const DebugLoc &DL) const {
7581 Register OpReg = Op.getReg();
7582 unsigned OpSubReg = Op.getSubReg();
7583
7584 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
7585 RI.getRegClassForReg(MRI, OpReg), OpSubReg);
7586
7587 // Check if operand is already the correct register class.
7588 if (DstRC == OpRC)
7589 return;
7590
7591 Register DstReg = MRI.createVirtualRegister(DstRC);
7592 auto Copy = BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg)
7593 .addReg(OpReg, {}, OpSubReg);
7594 Op.setReg(DstReg);
7595 Op.setSubReg(AMDGPU::NoSubRegister);
7596
7597 MachineInstr *Def = MRI.getVRegDef(OpReg);
7598 if (!Def)
7599 return;
7600
7601 // Try to eliminate the copy if it is copying an immediate value.
7602 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
7603 foldImmediate(*Copy, *Def, OpReg, &MRI);
7604
7605 bool ImpDef = Def->isImplicitDef();
7606 while (!ImpDef && Def && Def->isCopy()) {
7607 if (Def->getOperand(1).getReg().isPhysical())
7608 break;
7609 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
7610 ImpDef = Def && Def->isImplicitDef();
7611 }
7612 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
7613 !ImpDef)
7614 Copy.addReg(AMDGPU::EXEC, RegState::Implicit);
7615}
7616
7617// Emit the actual waterfall loop, executing the wrapped instruction for each
7618// unique value of \p ScalarOps across all lanes. In the best case we execute 1
7619// iteration, in the worst case we execute 64 (once per lane).
7622 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
7623 ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
7624 MachineFunction &MF = *LoopBB.getParent();
7626 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7628 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7629
7630 // Emit v_cmpx_eq and s_andn2_wrexec when both instructions are
7631 // available. Otherwise, use the previous pattern of v_cmp_eq,
7632 // s_and_saveexec, and s_xor.
7633 bool UseNewExecInstructions =
7634 ST.hasNoSdstCMPX() && TII.pseudoToMCOpcode(LMC.AndN2WrExecOpc) != -1;
7635
7637 Register CondReg;
7638
7639 Register PhiExec;
7640 Register NewExec;
7641
7642 if (UseNewExecInstructions) {
7643 PhiExec = MRI.createVirtualRegister(BoolXExecRC);
7644 NewExec = MRI.createVirtualRegister(BoolXExecRC);
7645 Register InitExec = MRI.createVirtualRegister(BoolXExecRC);
7646 BuildMI(PredBB, PredBB.end(), DL, TII.get(LMC.MovOpc), InitExec)
7647 .addReg(LMC.ExecReg);
7648
7649 BuildMI(LoopBB, I, DL, TII.get(TargetOpcode::PHI), PhiExec)
7650 .addReg(InitExec)
7651 .addMBB(&PredBB)
7652 .addReg(NewExec)
7653 .addMBB(&BodyBB);
7654 }
7655
7656 // Placement of v_cmpx instructions (when index is longer than 64 bit)
7657 // involves a trade-off between register pressure and latency:
7658 // (a) Defering all v_cmpx after all v_readfirstlane may increase
7659 // register pressure because arguments and results of all
7660 // v_readfirstlane instructions must stay live until deferred v_cmpx use them.
7661 // (b) Interleaving v_cmpx with v_readfirstlanes may reduce live ranges and
7662 // increase latency by placing v_readfirstlane instructions
7663 // immediately before v_cmpx instruction that directly depend on it.
7664 ///
7665 // Emitting interleaved v_cmpx and v_readfirstlane requires
7666 // block splitting because v_cmpx changes EXEC mask and therefore for safety
7667 // v_cmpx needs to be treated as terminator until after register allocation
7668 // (spill placement) and instruction reordering.
7669 //
7670 // Current implementation defers v_cmpx and leaves other instruction
7671 // scheduling decisions to later passes, where register pressure is known or
7672 // easier to approximate.
7673 // Non-terminators (V_READFIRSTLANE and REG_SEQUENCE) are inserted before I;
7674 // v_cmpx instructions are inserted at the end of LoopBB.
7675 // After the first v_cmpx is emitted, I is updated to point to it
7676 // so subsequent non-terminators are inserted before all v_cmpx instructions.
7677 for (auto [Idx, ScalarOp] : enumerate(ScalarOps)) {
7678 unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);
7679 unsigned NumSubRegs = RegSize / 32;
7680 Register VScalarOp = ScalarOp->getReg();
7681
7682 const TargetRegisterClass *RFLSrcRC =
7683 TII.getRegClass(TII.get(AMDGPU::V_READFIRSTLANE_B32), 1);
7684
7685 if (NumSubRegs == 1) {
7686 const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(VScalarOp);
7687 if (const TargetRegisterClass *Common =
7688 TRI->getCommonSubClass(VScalarOpRC, RFLSrcRC);
7689 Common != VScalarOpRC) {
7690 Register VRReg = MRI.createVirtualRegister(Common);
7691 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::COPY), VRReg).addReg(VScalarOp);
7692 VScalarOp = VRReg;
7693 }
7694 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7695
7696 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
7697 .addReg(VScalarOp);
7698
7699 if (UseNewExecInstructions) {
7700 auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
7701 TII.get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
7702 .addReg(CurReg)
7703 .addReg(VScalarOp);
7704 if (I == LoopBB.end())
7705 I = CmpxMI.getInstr()->getIterator();
7706 } else {
7707 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
7708
7709 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
7710 .addReg(CurReg)
7711 .addReg(VScalarOp);
7712
7713 // Combine the comparison results with AND.
7714 if (!CondReg) { // First.
7715 CondReg = NewCondReg;
7716 } else { // If not the first, we create an AND.
7717 Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
7718 BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
7719 .addReg(CondReg)
7720 .addReg(NewCondReg)
7721 .setOperandDead(3);
7722 CondReg = AndReg;
7723 }
7724 }
7725
7726 // Update ScalarOp operand to use the SGPR ScalarOp.
7727 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7728 ScalarOp->setReg(CurReg);
7729 else {
7730 // Insert into the same block of use
7731 BuildMI(*ScalarOp->getParent()->getParent(), ScalarOp->getParent(), DL,
7732 TII.get(AMDGPU::COPY), PhySGPRs[Idx])
7733 .addReg(CurReg);
7734 ScalarOp->setReg(PhySGPRs[Idx]);
7735 }
7736 } else {
7737 SmallVector<Register, 8> ReadlanePieces;
7738 RegState VScalarOpUndef = getUndefRegState(ScalarOp->isUndef());
7739 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
7740 "Unhandled register size");
7741
7742 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
7743 Register CurRegLo =
7744 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7745 Register CurRegHi =
7746 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7747
7748 // Read the next variant <- also loop target.
7749 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
7750 .addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx));
7751
7752 // Read the next variant <- also loop target.
7753 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
7754 .addReg(VScalarOp, VScalarOpUndef,
7755 TRI->getSubRegFromChannel(Idx + 1));
7756
7757 ReadlanePieces.push_back(CurRegLo);
7758 ReadlanePieces.push_back(CurRegHi);
7759
7760 // Comparison is to be done as 64-bit.
7761 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
7762 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
7763 .addReg(CurRegLo)
7764 .addImm(AMDGPU::sub0)
7765 .addReg(CurRegHi)
7766 .addImm(AMDGPU::sub1);
7767
7768 unsigned SubReg =
7769 NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Idx, 2);
7770
7771 if (UseNewExecInstructions) {
7772 auto CmpxMI = BuildMI(LoopBB, LoopBB.end(), DL,
7773 TII.get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
7774 .addReg(CurReg)
7775 .addReg(VScalarOp, VScalarOpUndef, SubReg);
7776 if (I == LoopBB.end())
7777 I = CmpxMI.getInstr()->getIterator();
7778 } else {
7779 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
7780 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
7781 .addReg(CurReg)
7782 .addReg(VScalarOp, VScalarOpUndef, SubReg);
7783
7784 // Combine the comparison results with AND.
7785 if (!CondReg) { // First.
7786 CondReg = NewCondReg;
7787 } else { // If not the first, we create an AND.
7788 Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
7789 BuildMI(LoopBB, I, DL, TII.get(LMC.AndOpc), AndReg)
7790 .addReg(CondReg)
7791 .addReg(NewCondReg)
7792 .setOperandDead(3);
7793 CondReg = AndReg;
7794 }
7795 }
7796 } // End for loop.
7797
7798 const auto *SScalarOpRC =
7799 TRI->getEquivalentSGPRClass(MRI.getRegClass(VScalarOp));
7800 Register SScalarOp = MRI.createVirtualRegister(SScalarOpRC);
7801
7802 // Build scalar ScalarOp.
7803 auto Merge =
7804 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
7805 unsigned Channel = 0;
7806 for (Register Piece : ReadlanePieces) {
7807 Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel++));
7808 }
7809
7810 // Update ScalarOp operand to use the SGPR ScalarOp.
7811 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7812 ScalarOp->setReg(SScalarOp);
7813 else {
7814 BuildMI(*ScalarOp->getParent()->getParent(), ScalarOp->getParent(), DL,
7815 TII.get(AMDGPU::COPY), PhySGPRs[Idx])
7816 .addReg(SScalarOp);
7817 ScalarOp->setReg(PhySGPRs[Idx]);
7818 }
7819 }
7820 }
7821
7822 // AndSaveExecOpc modifies EXEC but can't be isTerminator=1: terminators
7823 // that define virtual registers aren't supported.
7824 Register SaveExec;
7825 if (!UseNewExecInstructions) {
7826 SaveExec = MRI.createVirtualRegister(BoolXExecRC);
7827 MRI.setSimpleHint(SaveExec, CondReg);
7828
7829 // Update EXEC to matching lanes, saving original to SaveExec.
7830 BuildMI(LoopBB, I, DL, TII.get(LMC.AndSaveExecOpc), SaveExec)
7831 .addReg(CondReg)
7832 .setOperandDead(3);
7833 }
7834
7835 // The original instruction is here; we insert the terminators after it.
7836 I = BodyBB.end();
7837
7838 if (UseNewExecInstructions) {
7839 // Compute the remaining lanes into a plain virtual register and write EXEC
7840 // from a terminator, so spill code for NewExec is placed before EXEC
7841 // changes. SIOptimizeExecMasking opportunistically folds the pair back
7842 // into S_ANDN2_WREXEC after register allocation.
7843 MRI.setSimpleHint(NewExec, PhiExec);
7844 BuildMI(BodyBB, I, DL, TII.get(LMC.AndN2Opc), NewExec)
7845 .addReg(PhiExec)
7846 .addReg(LMC.ExecReg)
7847 .setOperandDead(3);
7848 BuildMI(BodyBB, I, DL, TII.get(LMC.MovTermOpc), LMC.ExecReg)
7849 .addReg(NewExec);
7850 } else {
7851 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
7852 BuildMI(BodyBB, I, DL, TII.get(LMC.XorTermOpc), LMC.ExecReg)
7853 .addReg(LMC.ExecReg)
7854 .addReg(SaveExec)
7855 .setOperandDead(3);
7856 }
7857
7858 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
7859}
7860
7861// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
7862// with SGPRs by iterating over all unique values across all lanes.
7863// Returns the loop basic block that now contains \p MI.
7864static MachineBasicBlock *
7868 MachineBasicBlock::iterator Begin = nullptr,
7870 ArrayRef<Register> PhySGPRs = {}) {
7871 assert((PhySGPRs.empty() || PhySGPRs.size() == ScalarOps.size()) &&
7872 "Physical SGPRs must be empty or match the number of scalar operands");
7874 MachineFunction &MF = *MBB.getParent();
7876 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7878 if (!Begin.isValid())
7879 Begin = &MI;
7880 if (!End.isValid()) {
7881 End = &MI;
7882 ++End;
7883 }
7884 const DebugLoc &DL = MI.getDebugLoc();
7886 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7887
7888 // Save SCC. Waterfall Loop may overwrite SCC.
7889 Register SaveSCCReg;
7890
7891 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk
7892 // rather than unlimited scan everywhere
7893 bool SCCNotDead =
7894 MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI,
7895 std::numeric_limits<unsigned>::max()) !=
7897 if (SCCNotDead) {
7898 SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7899 BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg)
7900 .addImm(1)
7901 .addImm(0);
7902 }
7903
7904 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
7905
7906 // Save the EXEC mask
7907 BuildMI(MBB, Begin, DL, TII.get(LMC.MovOpc), SaveExec).addReg(LMC.ExecReg);
7908
7909 // To insert the loop we need to split the block. Move everything after this
7910 // point to a new block, and insert a new empty block between the two.
7913 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
7915 ++MBBI;
7916
7917 MF.insert(MBBI, LoopBB);
7918 MF.insert(MBBI, BodyBB);
7919 MF.insert(MBBI, RemainderBB);
7920
7921 LoopBB->addSuccessor(BodyBB);
7922 BodyBB->addSuccessor(LoopBB);
7923 BodyBB->addSuccessor(RemainderBB);
7924
7925 // Move Begin to MI to the BodyBB, and the remainder of the block to
7926 // RemainderBB.
7927 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
7928 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end());
7929 BodyBB->splice(BodyBB->begin(), &MBB, Begin, MBB.end());
7930
7931 MBB.addSuccessor(LoopBB);
7932
7933 // Update dominators. We know that MBB immediately dominates LoopBB, that
7934 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
7935 // RemainderBB. RemainderBB immediately dominates all of the successors
7936 // transferred to it from MBB that MBB used to properly dominate.
7937 if (MDT) {
7938 MDT->addNewBlock(LoopBB, &MBB);
7939 MDT->addNewBlock(BodyBB, LoopBB);
7940 MDT->addNewBlock(RemainderBB, BodyBB);
7941 for (auto &Succ : RemainderBB->successors()) {
7942 if (MDT->properlyDominates(&MBB, Succ)) {
7943 MDT->changeImmediateDominator(Succ, RemainderBB);
7944 }
7945 }
7946 }
7947
7948 emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps,
7949 PhySGPRs);
7950
7951 MachineBasicBlock::iterator First = RemainderBB->begin();
7952 // Restore SCC
7953 if (SCCNotDead) {
7954 BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32))
7955 .addReg(SaveSCCReg)
7956 .addImm(0);
7957 }
7958
7959 // Restore the EXEC mask
7960 BuildMI(*RemainderBB, First, DL, TII.get(LMC.MovOpc), LMC.ExecReg)
7961 .addReg(SaveExec);
7962 return BodyBB;
7963}
7964
7965// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
7966static std::tuple<unsigned, unsigned>
7968 MachineBasicBlock &MBB = *MI.getParent();
7969 MachineFunction &MF = *MBB.getParent();
7971
7972 // Extract the ptr from the resource descriptor.
7973 unsigned RsrcPtr =
7974 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
7975 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
7976
7977 // Create an empty resource descriptor
7978 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
7979 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
7980 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
7981 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
7982 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
7983
7984 // Zero64 = 0
7985 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
7986 .addImm(0);
7987
7988 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
7989 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
7990 .addImm(Lo_32(RsrcDataFormat));
7991
7992 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
7993 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
7994 .addImm(Hi_32(RsrcDataFormat));
7995
7996 // NewSRsrc = {Zero64, SRsrcFormat}
7997 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
7998 .addReg(Zero64)
7999 .addImm(AMDGPU::sub0_sub1)
8000 .addReg(SRsrcFormatLo)
8001 .addImm(AMDGPU::sub2)
8002 .addReg(SRsrcFormatHi)
8003 .addImm(AMDGPU::sub3);
8004
8005 return std::tuple(RsrcPtr, NewSRsrc);
8006}
8007
8010 MachineDominatorTree *MDT) const {
8011 MachineFunction &MF = *MI.getMF();
8013 MachineBasicBlock *CreatedBB = nullptr;
8014
8015 // Legalize True16
8016 if (ST.useRealTrue16Insts())
8018
8019 // Legalize VOP2
8020 if (isVOP2(MI) || isVOPC(MI)) {
8022 return CreatedBB;
8023 }
8024
8025 // Legalize VOP3
8026 if (isVOP3(MI)) {
8028 return CreatedBB;
8029 }
8030
8031 // Legalize SMRD
8032 if (isSMRD(MI)) {
8034 return CreatedBB;
8035 }
8036
8037 // Legalize FLAT
8038 if (isFLAT(MI)) {
8040 return CreatedBB;
8041 }
8042
8043 // The GPR_IDX form needs its index in an SGPR. For the movrel form, a
8044 // divergent M0 write is waterfalled where the copy into M0 is lowered.
8045 if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(&MI)) {
8046 if (!LdStIdx->isGPRIdx())
8047 return CreatedBB;
8048 MachineOperand *Idx = &LdStIdx->getIdxOp();
8049 if (!RI.isSGPRReg(MRI, Idx->getReg()))
8050 CreatedBB = generateWaterFallLoop(*this, MI, {Idx}, MDT);
8051 return CreatedBB;
8052 }
8053
8054 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
8055 // VGPR dest type and SGPR sources, insert copies so all operands are
8056 // VGPRs. This seems to help operand folding / the register coalescer.
8057 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
8058 MachineBasicBlock *MBB = MI.getParent();
8059 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
8060 if (RI.hasVGPRs(DstRC)) {
8061 // Update all the operands so they are VGPR register classes. These may
8062 // not be the same register class because REG_SEQUENCE supports mixing
8063 // subregister index types e.g. sub0_sub1 + sub2 + sub3
8064 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
8065 MachineOperand &Op = MI.getOperand(I);
8066 if (!Op.isReg() || !Op.getReg().isVirtual())
8067 continue;
8068
8069 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
8070 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
8071 if (VRC == OpRC)
8072 continue;
8073
8074 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
8075 }
8076 }
8077
8078 return CreatedBB;
8079 }
8080
8081 // Legalize INSERT_SUBREG
8082 // src0 must have the same register class as dst
8083 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
8084 Register Dst = MI.getOperand(0).getReg();
8085 Register Src0 = MI.getOperand(1).getReg();
8086 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
8087 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
8088 if (DstRC != Src0RC) {
8089 MachineBasicBlock *MBB = MI.getParent();
8090 MachineOperand &Op = MI.getOperand(1);
8091 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
8092 }
8093 return CreatedBB;
8094 }
8095
8096 // Legalize SI_INIT_M0
8097 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
8098 MachineOperand &Src = MI.getOperand(0);
8099 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
8100 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
8101 return CreatedBB;
8102 }
8103
8104 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
8105 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
8106 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
8107 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
8108 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
8109 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
8110 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
8111 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
8112 MachineOperand &Src = MI.getOperand(1);
8113 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
8114 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
8115 return CreatedBB;
8116 }
8117
8118 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
8119 //
8120 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
8121 // scratch memory access. In both cases, the legalization never involves
8122 // conversion to the addr64 form.
8124 (isMUBUF(MI) || isMTBUF(MI)))) {
8125 AMDGPU::OpName RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI))
8126 ? AMDGPU::OpName::rsrc
8127 : AMDGPU::OpName::srsrc;
8128 MachineOperand *SRsrc = getNamedOperand(MI, RSrcOpName);
8129 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg())))
8130 CreatedBB = generateWaterFallLoop(*this, MI, {SRsrc}, MDT);
8131
8132 AMDGPU::OpName SampOpName =
8133 isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
8134 MachineOperand *SSamp = getNamedOperand(MI, SampOpName);
8135 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg())))
8136 CreatedBB = generateWaterFallLoop(*this, MI, {SSamp}, MDT);
8137
8138 return CreatedBB;
8139 }
8140
8141 // Legalize SI_CALL
8142 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
8143 MachineOperand *Dest = &MI.getOperand(0);
8144 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) {
8145 createWaterFallForSiCall(&MI, MDT, {Dest});
8146 }
8147 }
8148
8149 // Legalize s_sleep_var.
8150 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
8151 const DebugLoc &DL = MI.getDebugLoc();
8152 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8153 int Src0Idx =
8154 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
8155 MachineOperand &Src0 = MI.getOperand(Src0Idx);
8156 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
8157 .add(Src0);
8158 Src0.ChangeToRegister(Reg, false);
8159 return nullptr;
8160 }
8161
8162 // Legalize TENSOR_LOAD_TO_LDS_d2/_d4, TENSOR_STORE_FROM_LDS_d2/_d4. All their
8163 // operands are scalar.
8164 if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d2 ||
8165 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d4 ||
8166 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d2 ||
8167 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d4) {
8168 for (MachineOperand &Src : MI.explicit_operands()) {
8169 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
8170 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
8171 }
8172 return CreatedBB;
8173 }
8174
8175 // Legalize MUBUF instructions.
8176 bool isSoffsetLegal = true;
8177 int SoffsetIdx =
8178 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset);
8179 if (SoffsetIdx != -1) {
8180 MachineOperand *Soffset = &MI.getOperand(SoffsetIdx);
8181 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
8182 !RI.isSGPRClass(MRI.getRegClass(Soffset->getReg()))) {
8183 isSoffsetLegal = false;
8184 }
8185 }
8186
8187 bool isRsrcLegal = true;
8188 int RsrcIdx =
8189 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
8190 if (RsrcIdx != -1) {
8191 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
8192 if (Rsrc->isReg() && !RI.isSGPRReg(MRI, Rsrc->getReg()))
8193 isRsrcLegal = false;
8194 }
8195
8196 // The operands are legal.
8197 if (isRsrcLegal && isSoffsetLegal)
8198 return CreatedBB;
8199
8200 if (!isRsrcLegal) {
8201 // Legalize a VGPR Rsrc
8202 //
8203 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
8204 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
8205 // a zero-value SRsrc.
8206 //
8207 // If the instruction is _OFFSET (both idxen and offen disabled), and we
8208 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
8209 // above.
8210 //
8211 // Otherwise we are on non-ADDR64 hardware, and/or we have
8212 // idxen/offen/bothen and we fall back to a waterfall loop.
8213
8214 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
8215 MachineBasicBlock &MBB = *MI.getParent();
8216
8217 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
8218 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
8219 // This is already an ADDR64 instruction so we need to add the pointer
8220 // extracted from the resource descriptor to the current value of VAddr.
8221 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8222 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8223 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8224
8225 const auto *BoolXExecRC = RI.getWaveMaskRegClass();
8226 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
8227 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
8228
8229 unsigned RsrcPtr, NewSRsrc;
8230 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
8231
8232 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
8233 const DebugLoc &DL = MI.getDebugLoc();
8234 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo)
8235 .addDef(CondReg0)
8236 .addReg(RsrcPtr, {}, AMDGPU::sub0)
8237 .addReg(VAddr->getReg(), {}, AMDGPU::sub0)
8238 .addImm(0);
8239
8240 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
8241 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
8242 .addDef(CondReg1, RegState::Dead)
8243 .addReg(RsrcPtr, {}, AMDGPU::sub1)
8244 .addReg(VAddr->getReg(), {}, AMDGPU::sub1)
8245 .addReg(CondReg0)
8246 .addImm(0);
8247
8248 // NewVaddr = {NewVaddrHi, NewVaddrLo}
8249 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
8250 .addReg(NewVAddrLo)
8251 .addImm(AMDGPU::sub0)
8252 .addReg(NewVAddrHi)
8253 .addImm(AMDGPU::sub1);
8254
8255 VAddr->setReg(NewVAddr);
8256 Rsrc->setReg(NewSRsrc);
8257 } else if (!VAddr && ST.hasAddr64()) {
8258 // This instructions is the _OFFSET variant, so we need to convert it to
8259 // ADDR64.
8260 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
8261 "FIXME: Need to emit flat atomics here");
8262
8263 unsigned RsrcPtr, NewSRsrc;
8264 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
8265
8266 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8267 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
8268 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
8269 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
8270 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
8271
8272 // Atomics with return have an additional tied operand and are
8273 // missing some of the special bits.
8274 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
8275 MachineInstr *Addr64;
8276
8277 if (!VDataIn) {
8278 // Regular buffer load / store.
8280 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
8281 .add(*VData)
8282 .addReg(NewVAddr)
8283 .addReg(NewSRsrc)
8284 .add(*SOffset)
8285 .add(*Offset);
8286
8287 if (const MachineOperand *CPol =
8288 getNamedOperand(MI, AMDGPU::OpName::cpol)) {
8289 MIB.addImm(CPol->getImm());
8290 }
8291
8292 if (const MachineOperand *TFE =
8293 getNamedOperand(MI, AMDGPU::OpName::tfe)) {
8294 MIB.addImm(TFE->getImm());
8295 }
8296
8297 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
8298
8299 MIB.cloneMemRefs(MI);
8300 Addr64 = MIB;
8301 } else {
8302 // Atomics with return.
8303 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
8304 .add(*VData)
8305 .add(*VDataIn)
8306 .addReg(NewVAddr)
8307 .addReg(NewSRsrc)
8308 .add(*SOffset)
8309 .add(*Offset)
8310 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol))
8311 .cloneMemRefs(MI);
8312 }
8313
8314 MI.removeFromParent();
8315
8316 // NewVaddr = {NewVaddrHi, NewVaddrLo}
8317 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
8318 NewVAddr)
8319 .addReg(RsrcPtr, {}, AMDGPU::sub0)
8320 .addImm(AMDGPU::sub0)
8321 .addReg(RsrcPtr, {}, AMDGPU::sub1)
8322 .addImm(AMDGPU::sub1);
8323 } else {
8324 // Legalize a VGPR Rsrc and soffset together.
8325 if (!isSoffsetLegal) {
8326 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
8327 CreatedBB = generateWaterFallLoop(*this, MI, {Rsrc, Soffset}, MDT);
8328 return CreatedBB;
8329 }
8330 CreatedBB = generateWaterFallLoop(*this, MI, {Rsrc}, MDT);
8331 return CreatedBB;
8332 }
8333 }
8334
8335 // Legalize a VGPR soffset.
8336 if (!isSoffsetLegal) {
8337 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
8338 CreatedBB = generateWaterFallLoop(*this, MI, {Soffset}, MDT);
8339 return CreatedBB;
8340 }
8341 return CreatedBB;
8342}
8343
8345 if (InSet.insert(MI).second)
8346 InstrList.push_back(MI);
8347 // Add MBUF instructiosn to deferred list.
8348 int RsrcIdx =
8349 AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc);
8350 if (RsrcIdx != -1) {
8351 DeferredList.insert(MI);
8352 }
8353}
8354
8356 return DeferredList.contains(MI);
8357}
8358
8359// Legalize size mismatches between 16bit and 32bit registers in v2s copy
8360// lowering (change sgpr to vgpr).
8361// This is mainly caused by 16bit SALU and 16bit VALU using reg with different
8362// size. Need to legalize the size of the operands during the vgpr lowering
8363// chain. This can be removed after we have sgpr16 in place
8365 MachineRegisterInfo &MRI) const {
8366 if (!ST.useRealTrue16Insts())
8367 return;
8368
8369 unsigned Opcode = MI.getOpcode();
8370 MachineBasicBlock *MBB = MI.getParent();
8371 // Legalize operands and check for size mismatch
8372 if (OpIdx >= MI.getNumExplicitOperands() ||
8373 OpIdx >= get(Opcode).getNumOperands() ||
8374 get(Opcode).operands()[OpIdx].RegClass == -1)
8375 return;
8376
8377 MachineOperand &Op = MI.getOperand(OpIdx);
8378 if (!Op.isReg() || !Op.getReg().isVirtual() || Op.isDef())
8379 return;
8380
8381 const TargetRegisterClass *CurrRC = MRI.getRegClass(Op.getReg());
8382 if (!RI.isVGPRClass(CurrRC))
8383 return;
8384
8385 int16_t RCID = getOpRegClassID(get(Opcode).operands()[OpIdx]);
8386 const TargetRegisterClass *ExpectedRC = RI.getRegClass(RCID);
8387 if (RI.getMatchingSuperRegClass(CurrRC, ExpectedRC, AMDGPU::lo16)) {
8388 // Default to the lo16 only if the subregister is not specified.
8389 if (Op.getSubReg() == AMDGPU::NoSubRegister)
8390 Op.setSubReg(AMDGPU::lo16);
8391 return;
8392 }
8393
8394 const TargetRegisterClass *CurrSRC =
8395 RI.getSubRegisterClass(CurrRC, Op.getSubReg());
8396 if (RI.getMatchingSuperRegClass(ExpectedRC, CurrSRC, AMDGPU::lo16)) {
8397 Op.setReg(buildVGPR32FromLo16(*MBB, MI, MI.getDebugLoc(), Op.getReg(),
8398 Op.getSubReg()));
8399 Op.setSubReg(AMDGPU::NoSubRegister);
8400 }
8401}
8403 MachineRegisterInfo &MRI) const {
8404 for (unsigned OpIdx = 0; OpIdx < MI.getNumExplicitOperands(); OpIdx++)
8406}
8407
8410 const DebugLoc &DL, Register Src,
8411 unsigned SrcSubReg) const {
8412 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8413 Register Dst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8414 Register Undef = MRI.createVirtualRegister(&AMDGPU::VGPR_16RegClass);
8415 BuildMI(MBB, I, DL, get(AMDGPU::IMPLICIT_DEF), Undef);
8416 BuildMI(MBB, I, DL, get(AMDGPU::REG_SEQUENCE), Dst)
8417 .addReg(Src, {}, SrcSubReg)
8418 .addImm(AMDGPU::lo16)
8419 .addReg(Undef)
8420 .addImm(AMDGPU::hi16);
8421 return Dst;
8422}
8423
8427 ArrayRef<Register> PhySGPRs) const {
8428 assert(MI->getOpcode() == AMDGPU::SI_CALL_ISEL &&
8429 "This only handle waterfall for SI_CALL_ISEL");
8430 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
8431 // following copies, we also need to move copies from and to physical
8432 // registers into the loop block.
8433 // Also move the copies to physical registers into the loop block
8434 MachineBasicBlock &MBB = *MI->getParent();
8436 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
8437 --Start;
8439 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
8440 ++End;
8441
8442 // Also include following copies of the return value
8443 ++End;
8444 while (End != MBB.end() && End->isCopy() &&
8445 MI->definesRegister(End->getOperand(1).getReg(), &RI))
8446 ++End;
8447
8448 generateWaterFallLoop(*this, *MI, ScalarOps, MDT, Start, End, PhySGPRs);
8449}
8450
8452 MachineDominatorTree *MDT) const {
8454 DenseMap<MachineInstr *, bool> V2SPhyCopiesToErase;
8455 while (!Worklist.empty()) {
8456 MachineInstr &Inst = *Worklist.top();
8457 Worklist.erase_top();
8458 // Skip MachineInstr in the deferred list.
8459 if (Worklist.isDeferred(&Inst))
8460 continue;
8461 moveToVALUImpl(Worklist, MDT, Inst, WaterFalls, V2SPhyCopiesToErase);
8462 }
8463
8464 // Deferred list of instructions will be processed once
8465 // all the MachineInstr in the worklist are done.
8466 for (MachineInstr *Inst : Worklist.getDeferredList()) {
8467 moveToVALUImpl(Worklist, MDT, *Inst, WaterFalls, V2SPhyCopiesToErase);
8468 assert(Worklist.empty() &&
8469 "Deferred MachineInstr are not supposed to re-populate worklist");
8470 }
8471
8472 for (auto &Entry : WaterFalls) {
8473 if (Entry.first->getOpcode() == AMDGPU::SI_CALL_ISEL)
8474 createWaterFallForSiCall(Entry.first, MDT, Entry.second.MOs,
8475 Entry.second.SGPRs);
8476 }
8477
8478 for (std::pair<MachineInstr *, bool> Entry : V2SPhyCopiesToErase)
8479 if (Entry.second)
8480 Entry.first->eraseFromParent();
8481}
8483 MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const {
8484 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
8485 // hope for the best.
8486 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI, DstReg);
8487 ArrayRef<int16_t> SubRegIndices = RI.getRegSplitParts(DstRC, 4);
8488 if (SubRegIndices.size() <= 1) {
8489 Register NewDst = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8490 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),
8491 get(AMDGPU::V_READFIRSTLANE_B32), NewDst)
8492 .add(Inst.getOperand(1));
8493 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY),
8494 DstReg)
8495 .addReg(NewDst);
8496 } else {
8498 for (int16_t Indice : SubRegIndices) {
8499 Register NewDst = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8500 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),
8501 get(AMDGPU::V_READFIRSTLANE_B32), NewDst)
8502 .addReg(Inst.getOperand(1).getReg(), {}, Indice);
8503
8504 DstRegs.push_back(NewDst);
8505 }
8507 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),
8508 get(AMDGPU::REG_SEQUENCE), DstReg);
8509 for (unsigned i = 0; i < SubRegIndices.size(); ++i) {
8510 MIB.addReg(DstRegs[i]);
8511 MIB.addImm(RI.getSubRegFromChannel(i));
8512 }
8513 }
8514}
8515
8516// M0 holds one value, but the VGPR copied into it holds one per lane. A movrel
8517// access adds M0 to its base register, so each one that reads the copy is
8518// wrapped in a waterfall loop that puts the value of each group of lanes in M0
8519// in turn. Every other reader gets the first lane, as all of them do when no
8520// access reads the copy.
8524 MachineDominatorTree *MDT) {
8525 auto IsMovrelAccess = [](const MachineInstr *MI) {
8526 const auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(MI);
8527 return LdStIdx && !LdStIdx->isGPRIdx();
8528 };
8529 if (none_of(Readers, IsMovrelAccess)) {
8530 TII.createReadFirstLaneFromCopyToPhysReg(MRI, AMDGPU::M0, Copy);
8531 return;
8532 }
8533
8534 assert(!Copy.getOperand(1).getSubReg() &&
8535 "selection copies a whole register into M0");
8536 Register Idx = Copy.getOperand(1).getReg();
8537 Register FirstLane;
8538 for (MachineInstr *Reader : Readers) {
8539 if (IsMovrelAccess(Reader)) {
8540 MachineOperand *M0Use =
8541 Reader->findRegisterUseOperand(AMDGPU::M0, &TII.getRegisterInfo());
8542 M0Use->setReg(Idx);
8543 generateWaterFallLoop(TII, *Reader, M0Use, MDT, nullptr, nullptr,
8544 {AMDGPU::M0});
8545 continue;
8546 }
8547
8548 if (!FirstLane) {
8549 FirstLane = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8550 BuildMI(*Copy.getParent(), Copy, Copy.getDebugLoc(),
8551 TII.get(AMDGPU::V_READFIRSTLANE_B32), FirstLane)
8552 .addReg(Idx);
8553 }
8554 BuildMI(*Reader->getParent(), Reader, Reader->getDebugLoc(),
8555 TII.get(AMDGPU::COPY), AMDGPU::M0)
8556 .addReg(FirstLane);
8557 }
8558}
8559
8561 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
8564 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8565 // In SSA a copy into a physical register and its readers share a block, so
8566 // only the rest of this block is searched.
8568 for (MachineInstr &MI :
8570 Inst.getParent()->end())) {
8571 if (MI.readsRegister(DstReg, &RI))
8572 Readers.push_back(&MI);
8573
8574 if (MI.modifiesRegister(DstReg, &RI))
8575 break;
8576 }
8577
8578 if (DstReg == AMDGPU::M0) {
8579 lowerVGPRCopyToM0(*this, Inst, Readers, MRI, MDT);
8580 V2SPhyCopiesToErase.try_emplace(&Inst, true);
8581 return;
8582 }
8583
8584 Register SrcReg = Inst.getOperand(1).getReg();
8585 for (MachineInstr *UseMI : Readers) {
8586 // For SI_CALL_ISEL users, replace the phys SGPR with the VGPR source
8587 // and record the operand for later waterfall loop generation.
8588 if (UseMI->getOpcode() == AMDGPU::SI_CALL_ISEL) {
8589 for (unsigned i = 0; i < UseMI->getNumOperands(); ++i) {
8590 if (UseMI->getOperand(i).isReg() &&
8591 UseMI->getOperand(i).getReg() == DstReg) {
8592 MachineOperand *MO = &UseMI->getOperand(i);
8593 MO->setReg(SrcReg);
8594 V2PhysSCopyInfo &V2SCopyInfo = WaterFalls[UseMI];
8595 V2SCopyInfo.MOs.push_back(MO);
8596 V2SCopyInfo.SGPRs.push_back(DstReg);
8597 V2SPhyCopiesToErase.try_emplace(&Inst, true);
8598 }
8599 }
8600 } else if (UseMI->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG &&
8601 UseMI->getOperand(0).isReg() &&
8602 UseMI->getOperand(0).getReg() == DstReg) {
8604 V2SPhyCopiesToErase.try_emplace(&Inst, true);
8605 } else {
8606 // COPY cannot be erased if other type of inst uses it.
8607 V2SPhyCopiesToErase[&Inst] = false;
8608 }
8609 }
8610}
8611
8613 SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst,
8615 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8616
8618 if (!MBB)
8619 return;
8620 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
8621 unsigned Opcode = Inst.getOpcode();
8622 unsigned NewOpcode = getVALUOp(Inst);
8623 const DebugLoc &DL = Inst.getDebugLoc();
8624
8625 // Handle some special cases
8626 switch (Opcode) {
8627 default:
8628 break;
8629 case AMDGPU::S_ADD_I32:
8630 case AMDGPU::S_SUB_I32: {
8631 // FIXME: The u32 versions currently selected use the carry.
8632 bool Changed;
8633 MachineBasicBlock *CreatedBBTmp = nullptr;
8634 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
8635 if (Changed)
8636 return;
8637
8638 // Default handling
8639 break;
8640 }
8641
8642 case AMDGPU::S_MUL_U64:
8643 if (ST.useVMulU64Inst()) {
8644 NewOpcode = AMDGPU::V_MUL_U64_e64;
8645 break;
8646 }
8647 // Split s_mul_u64 in 32-bit vector multiplications.
8648 splitScalarSMulU64(Worklist, Inst, MDT);
8649 Inst.eraseFromParent();
8650 return;
8651
8652 case AMDGPU::S_MUL_U64_U32_PSEUDO:
8653 case AMDGPU::S_MUL_I64_I32_PSEUDO:
8654 // This is a special case of s_mul_u64 where all the operands are either
8655 // zero extended or sign extended.
8656 splitScalarSMulPseudo(Worklist, Inst, MDT);
8657 Inst.eraseFromParent();
8658 return;
8659
8660 case AMDGPU::S_AND_B64:
8661 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
8662 Inst.eraseFromParent();
8663 return;
8664
8665 case AMDGPU::S_OR_B64:
8666 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
8667 Inst.eraseFromParent();
8668 return;
8669
8670 case AMDGPU::S_XOR_B64:
8671 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
8672 Inst.eraseFromParent();
8673 return;
8674
8675 case AMDGPU::S_NAND_B64:
8676 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
8677 Inst.eraseFromParent();
8678 return;
8679
8680 case AMDGPU::S_NOR_B64:
8681 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
8682 Inst.eraseFromParent();
8683 return;
8684
8685 case AMDGPU::S_XNOR_B64:
8686 if (ST.hasDLInsts())
8687 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
8688 else
8689 splitScalar64BitXnor(Worklist, Inst, MDT);
8690 Inst.eraseFromParent();
8691 return;
8692
8693 case AMDGPU::S_ANDN2_B64:
8694 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
8695 Inst.eraseFromParent();
8696 return;
8697
8698 case AMDGPU::S_ORN2_B64:
8699 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
8700 Inst.eraseFromParent();
8701 return;
8702
8703 case AMDGPU::S_BREV_B64:
8704 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true);
8705 Inst.eraseFromParent();
8706 return;
8707
8708 case AMDGPU::S_NOT_B64:
8709 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
8710 Inst.eraseFromParent();
8711 return;
8712
8713 case AMDGPU::S_BCNT1_I32_B64:
8714 splitScalar64BitBCNT(Worklist, Inst);
8715 Inst.eraseFromParent();
8716 return;
8717
8718 case AMDGPU::S_BFE_I64:
8719 splitScalar64BitBFE(Worklist, Inst);
8720 Inst.eraseFromParent();
8721 return;
8722
8723 case AMDGPU::S_FLBIT_I32_B64:
8724 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32);
8725 Inst.eraseFromParent();
8726 return;
8727 case AMDGPU::S_FF1_I32_B64:
8728 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32);
8729 Inst.eraseFromParent();
8730 return;
8731
8732 case AMDGPU::S_LSHL_B32:
8733 if (ST.hasOnlyRevVALUShifts()) {
8734 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
8735 swapOperands(Inst);
8736 }
8737 break;
8738 case AMDGPU::S_ASHR_I32:
8739 if (ST.hasOnlyRevVALUShifts()) {
8740 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
8741 swapOperands(Inst);
8742 }
8743 break;
8744 case AMDGPU::S_LSHR_B32:
8745 if (ST.hasOnlyRevVALUShifts()) {
8746 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
8747 swapOperands(Inst);
8748 }
8749 break;
8750 case AMDGPU::S_LSHL_B64:
8751 if (ST.hasOnlyRevVALUShifts()) {
8752 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
8753 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
8754 : AMDGPU::V_LSHLREV_B64_e64;
8755 swapOperands(Inst);
8756 }
8757 break;
8758 case AMDGPU::S_ASHR_I64:
8759 if (ST.hasOnlyRevVALUShifts()) {
8760 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
8761 swapOperands(Inst);
8762 }
8763 break;
8764 case AMDGPU::S_LSHR_B64:
8765 if (ST.hasOnlyRevVALUShifts()) {
8766 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
8767 swapOperands(Inst);
8768 }
8769 break;
8770
8771 case AMDGPU::S_ABS_I32:
8772 lowerScalarAbs(Worklist, Inst);
8773 Inst.eraseFromParent();
8774 return;
8775
8776 case AMDGPU::S_ABSDIFF_I32:
8777 lowerScalarAbsDiff(Worklist, Inst);
8778 Inst.eraseFromParent();
8779 return;
8780
8781 case AMDGPU::S_CBRANCH_SCC0:
8782 case AMDGPU::S_CBRANCH_SCC1: {
8783 // Clear unused bits of vcc
8784 Register CondReg = Inst.getOperand(1).getReg();
8785 bool IsSCC = CondReg == AMDGPU::SCC;
8787 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(LMC.AndOpc), LMC.VccReg)
8788 .addReg(LMC.ExecReg)
8789 .addReg(IsSCC ? LMC.VccReg : CondReg)
8790 .setOperandDead(3); // implicit-def $scc
8791 Inst.removeOperand(1);
8792 } break;
8793
8794 case AMDGPU::S_BFE_U64:
8795 case AMDGPU::S_BFM_B64:
8796 llvm_unreachable("Moving this op to VALU not implemented");
8797
8798 case AMDGPU::S_PACK_LL_B32_B16:
8799 case AMDGPU::S_PACK_LH_B32_B16:
8800 case AMDGPU::S_PACK_HL_B32_B16:
8801 case AMDGPU::S_PACK_HH_B32_B16:
8802 movePackToVALU(Worklist, MRI, Inst);
8803 Inst.eraseFromParent();
8804 return;
8805
8806 case AMDGPU::S_XNOR_B32:
8807 lowerScalarXnor(Worklist, Inst);
8808 Inst.eraseFromParent();
8809 return;
8810
8811 case AMDGPU::S_NAND_B32:
8812 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
8813 Inst.eraseFromParent();
8814 return;
8815
8816 case AMDGPU::S_NOR_B32:
8817 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
8818 Inst.eraseFromParent();
8819 return;
8820
8821 case AMDGPU::S_ANDN2_B32:
8822 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
8823 Inst.eraseFromParent();
8824 return;
8825
8826 case AMDGPU::S_ORN2_B32:
8827 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
8828 Inst.eraseFromParent();
8829 return;
8830
8831 // TODO: remove as soon as everything is ready
8832 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
8833 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
8834 // can only be selected from the uniform SDNode.
8835 case AMDGPU::S_ADD_CO_PSEUDO:
8836 case AMDGPU::S_SUB_CO_PSEUDO: {
8837 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
8838 ? AMDGPU::V_ADDC_U32_e64
8839 : AMDGPU::V_SUBB_U32_e64;
8840 const auto *CarryRC = RI.getWaveMaskRegClass();
8841
8842 Register CarryInReg = Inst.getOperand(4).getReg();
8843 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) {
8844 Register NewCarryReg = MRI.createVirtualRegister(CarryRC);
8845 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg)
8846 .addReg(CarryInReg);
8847 }
8848
8849 Register CarryOutReg = Inst.getOperand(1).getReg();
8850
8851 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(
8852 MRI.getRegClass(Inst.getOperand(0).getReg())));
8853 MachineInstr *CarryOp =
8854 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg)
8855 .addReg(CarryOutReg, RegState::Define)
8856 .add(Inst.getOperand(2))
8857 .add(Inst.getOperand(3))
8858 .addReg(CarryInReg)
8859 .addImm(0);
8860 legalizeOperands(*CarryOp);
8861 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg);
8862 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist);
8863 Inst.eraseFromParent();
8864 }
8865 return;
8866 case AMDGPU::S_UADDO_PSEUDO:
8867 case AMDGPU::S_USUBO_PSEUDO: {
8868 MachineOperand &Dest0 = Inst.getOperand(0);
8869 MachineOperand &Dest1 = Inst.getOperand(1);
8870 MachineOperand &Src0 = Inst.getOperand(2);
8871 MachineOperand &Src1 = Inst.getOperand(3);
8872
8873 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
8874 ? AMDGPU::V_ADD_CO_U32_e64
8875 : AMDGPU::V_SUB_CO_U32_e64;
8876 const TargetRegisterClass *NewRC =
8877 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg()));
8878 Register DestReg = MRI.createVirtualRegister(NewRC);
8879 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg)
8880 .addReg(Dest1.getReg(), RegState::Define)
8881 .add(Src0)
8882 .add(Src1)
8883 .addImm(0); // clamp bit
8884
8885 legalizeOperands(*NewInstr, MDT);
8886 MRI.replaceRegWith(Dest0.getReg(), DestReg);
8887 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist);
8888 Inst.eraseFromParent();
8889 }
8890 return;
8891 case AMDGPU::S_LSHL1_ADD_U32:
8892 case AMDGPU::S_LSHL2_ADD_U32:
8893 case AMDGPU::S_LSHL3_ADD_U32:
8894 case AMDGPU::S_LSHL4_ADD_U32: {
8895 MachineOperand &Dest = Inst.getOperand(0);
8896 MachineOperand &Src0 = Inst.getOperand(1);
8897 MachineOperand &Src1 = Inst.getOperand(2);
8898 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 1
8899 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 2
8900 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 3
8901 : 4);
8902
8903 const TargetRegisterClass *NewRC =
8904 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest.getReg()));
8905 Register DestReg = MRI.createVirtualRegister(NewRC);
8906 MachineInstr *NewInstr =
8907 BuildMI(*MBB, &Inst, DL, get(AMDGPU::V_LSHL_ADD_U32_e64), DestReg)
8908 .add(Src0)
8909 .addImm(ShiftAmt)
8910 .add(Src1);
8911
8912 legalizeOperands(*NewInstr, MDT);
8913 MRI.replaceRegWith(Dest.getReg(), DestReg);
8914 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist);
8915 Inst.eraseFromParent();
8916 }
8917 return;
8918 case AMDGPU::S_CSELECT_B32:
8919 case AMDGPU::S_CSELECT_B64:
8920 lowerSelect(Worklist, Inst, MDT);
8921 Inst.eraseFromParent();
8922 return;
8923 case AMDGPU::S_CMP_EQ_I32:
8924 case AMDGPU::S_CMP_LG_I32:
8925 case AMDGPU::S_CMP_GT_I32:
8926 case AMDGPU::S_CMP_GE_I32:
8927 case AMDGPU::S_CMP_LT_I32:
8928 case AMDGPU::S_CMP_LE_I32:
8929 case AMDGPU::S_CMP_EQ_U32:
8930 case AMDGPU::S_CMP_LG_U32:
8931 case AMDGPU::S_CMP_GT_U32:
8932 case AMDGPU::S_CMP_GE_U32:
8933 case AMDGPU::S_CMP_LT_U32:
8934 case AMDGPU::S_CMP_LE_U32:
8935 case AMDGPU::S_CMP_EQ_U64:
8936 case AMDGPU::S_CMP_LG_U64:
8937 case AMDGPU::S_CMP_LT_F32:
8938 case AMDGPU::S_CMP_EQ_F32:
8939 case AMDGPU::S_CMP_LE_F32:
8940 case AMDGPU::S_CMP_GT_F32:
8941 case AMDGPU::S_CMP_LG_F32:
8942 case AMDGPU::S_CMP_GE_F32:
8943 case AMDGPU::S_CMP_O_F32:
8944 case AMDGPU::S_CMP_U_F32:
8945 case AMDGPU::S_CMP_NGE_F32:
8946 case AMDGPU::S_CMP_NLG_F32:
8947 case AMDGPU::S_CMP_NGT_F32:
8948 case AMDGPU::S_CMP_NLE_F32:
8949 case AMDGPU::S_CMP_NEQ_F32:
8950 case AMDGPU::S_CMP_NLT_F32: {
8951 Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass());
8952 auto NewInstr =
8953 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode), CondReg)
8954 .setMIFlags(Inst.getFlags());
8955 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src0_modifiers) >=
8956 0) {
8957 NewInstr
8958 .addImm(0) // src0_modifiers
8959 .add(Inst.getOperand(0)) // src0
8960 .addImm(0) // src1_modifiers
8961 .add(Inst.getOperand(1)) // src1
8962 .addImm(0); // clamp
8963 } else {
8964 NewInstr.add(Inst.getOperand(0)).add(Inst.getOperand(1));
8965 }
8966 legalizeOperands(*NewInstr, MDT);
8967 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC, /*TRI=*/nullptr);
8968 const MachineOperand &SCCOp = Inst.getOperand(SCCIdx);
8969 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg);
8970 Inst.eraseFromParent();
8971 return;
8972 }
8973 case AMDGPU::S_CMP_LT_F16:
8974 case AMDGPU::S_CMP_EQ_F16:
8975 case AMDGPU::S_CMP_LE_F16:
8976 case AMDGPU::S_CMP_GT_F16:
8977 case AMDGPU::S_CMP_LG_F16:
8978 case AMDGPU::S_CMP_GE_F16:
8979 case AMDGPU::S_CMP_O_F16:
8980 case AMDGPU::S_CMP_U_F16:
8981 case AMDGPU::S_CMP_NGE_F16:
8982 case AMDGPU::S_CMP_NLG_F16:
8983 case AMDGPU::S_CMP_NGT_F16:
8984 case AMDGPU::S_CMP_NLE_F16:
8985 case AMDGPU::S_CMP_NEQ_F16:
8986 case AMDGPU::S_CMP_NLT_F16: {
8987 Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass());
8988 auto NewInstr =
8989 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode), CondReg)
8990 .setMIFlags(Inst.getFlags());
8991 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0_modifiers)) {
8992 NewInstr
8993 .addImm(0) // src0_modifiers
8994 .add(Inst.getOperand(0)) // src0
8995 .addImm(0) // src1_modifiers
8996 .add(Inst.getOperand(1)) // src1
8997 .addImm(0); // clamp
8998 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::op_sel))
8999 NewInstr.addImm(0); // op_sel0
9000 } else {
9001 NewInstr
9002 .add(Inst.getOperand(0))
9003 .add(Inst.getOperand(1));
9004 }
9005 legalizeOperands(*NewInstr, MDT);
9006 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC, /*TRI=*/nullptr);
9007 const MachineOperand &SCCOp = Inst.getOperand(SCCIdx);
9008 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg);
9009 Inst.eraseFromParent();
9010 return;
9011 }
9012 case AMDGPU::S_CVT_HI_F32_F16: {
9013 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9014 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9015 if (ST.useRealTrue16Insts()) {
9016 BuildMI(*MBB, Inst, DL, get(AMDGPU::COPY), TmpReg)
9017 .add(Inst.getOperand(1));
9018 BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
9019 .addImm(0) // src0_modifiers
9020 .addReg(TmpReg, {}, AMDGPU::hi16)
9021 .addImm(0) // clamp
9022 .addImm(0) // omod
9023 .addImm(0); // op_sel0
9024 } else {
9025 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
9026 .addImm(16)
9027 .add(Inst.getOperand(1));
9028 BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
9029 .addImm(0) // src0_modifiers
9030 .addReg(TmpReg)
9031 .addImm(0) // clamp
9032 .addImm(0); // omod
9033 }
9034
9035 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
9036 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
9037 Inst.eraseFromParent();
9038 return;
9039 }
9040 case AMDGPU::S_MINIMUM_F32:
9041 case AMDGPU::S_MAXIMUM_F32: {
9042 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9043 MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
9044 .addImm(0) // src0_modifiers
9045 .add(Inst.getOperand(1))
9046 .addImm(0) // src1_modifiers
9047 .add(Inst.getOperand(2))
9048 .addImm(0) // clamp
9049 .addImm(0); // omod
9050 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
9051
9052 legalizeOperands(*NewInstr, MDT);
9053 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
9054 Inst.eraseFromParent();
9055 return;
9056 }
9057 case AMDGPU::S_MINIMUM_F16:
9058 case AMDGPU::S_MAXIMUM_F16: {
9059 Register NewDst = MRI.createVirtualRegister(ST.useRealTrue16Insts()
9060 ? &AMDGPU::VGPR_16RegClass
9061 : &AMDGPU::VGPR_32RegClass);
9062 MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
9063 .addImm(0) // src0_modifiers
9064 .add(Inst.getOperand(1))
9065 .addImm(0) // src1_modifiers
9066 .add(Inst.getOperand(2))
9067 .addImm(0) // clamp
9068 .addImm(0) // omod
9069 .addImm(0); // opsel0
9070 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
9071 legalizeOperands(*NewInstr, MDT);
9072 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
9073 Inst.eraseFromParent();
9074 return;
9075 }
9076 case AMDGPU::V_S_EXP_F16_e64:
9077 case AMDGPU::V_S_LOG_F16_e64:
9078 case AMDGPU::V_S_RCP_F16_e64:
9079 case AMDGPU::V_S_RSQ_F16_e64:
9080 case AMDGPU::V_S_SQRT_F16_e64: {
9081 Register NewDst = MRI.createVirtualRegister(ST.useRealTrue16Insts()
9082 ? &AMDGPU::VGPR_16RegClass
9083 : &AMDGPU::VGPR_32RegClass);
9084 auto NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
9085 .add(Inst.getOperand(1)) // src0_modifiers
9086 .add(Inst.getOperand(2))
9087 .add(Inst.getOperand(3)) // clamp
9088 .add(Inst.getOperand(4)) // omod
9089 .setMIFlags(Inst.getFlags());
9090 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::op_sel))
9091 NewInstr.addImm(0); // opsel0
9092 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
9093 legalizeOperands(*NewInstr, MDT);
9094 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
9095 Inst.eraseFromParent();
9096 return;
9097 }
9098 }
9099
9100 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
9101 // We cannot move this instruction to the VALU, so we should try to
9102 // legalize its operands instead.
9103 legalizeOperands(Inst, MDT);
9104 return;
9105 }
9106 // Handle converting generic instructions like COPY-to-SGPR into
9107 // COPY-to-VGPR.
9108 if (NewOpcode == Opcode) {
9109 Register DstReg = Inst.getOperand(0).getReg();
9110 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
9111
9112 if (Inst.isCopy() && DstReg.isPhysical() &&
9113 Inst.getOperand(1).getReg().isVirtual()) {
9114 handleCopyToPhysHelper(Worklist, DstReg, Inst, MRI, MDT, WaterFalls,
9115 V2SPhyCopiesToErase);
9116 return;
9117 }
9118
9119 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual()) {
9120 Register NewDstReg = Inst.getOperand(1).getReg();
9121 const TargetRegisterClass *SrcRC = RI.getRegClassForReg(MRI, NewDstReg);
9122 if (const TargetRegisterClass *CommonRC =
9123 RI.getCommonSubClass(NewDstRC, SrcRC)) {
9124 // Instead of creating a copy where src and dst are the same register
9125 // class, we just replace all uses of dst with src. These kinds of
9126 // copies interfere with the heuristics MachineSink uses to decide
9127 // whether or not to split a critical edge. Since the pass assumes
9128 // that copies will end up as machine instructions and not be
9129 // eliminated.
9130 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
9131 unsigned SrcSubReg = Inst.getOperand(1).getSubReg();
9132 bool IsUndef = Inst.getOperand(1).isUndef();
9133 for (MachineOperand &UseMO :
9134 make_early_inc_range(MRI.use_operands(DstReg))) {
9135 UseMO.setSubReg(
9136 RI.composeSubRegIndices(SrcSubReg, UseMO.getSubReg()));
9137 UseMO.setReg(NewDstReg);
9138 if (IsUndef)
9139 UseMO.setIsUndef();
9140 }
9141
9142 if (!MRI.constrainRegClass(NewDstReg, CommonRC))
9143 llvm_unreachable("failed to constrain register");
9144
9145 Inst.eraseFromParent();
9146
9147 for (MachineOperand &UseMO :
9148 make_early_inc_range(MRI.use_operands(NewDstReg))) {
9149 MachineInstr &UseMI = *UseMO.getParent();
9150
9151 // Legalize t16 operands since replaceReg is called after
9152 // addUsersToVALU.
9154
9155 unsigned OpIdx = UseMI.getOperandNo(&UseMO);
9156 if (const TargetRegisterClass *OpRC =
9157 getRegClass(UseMI.getDesc(), OpIdx))
9158 MRI.constrainRegClass(NewDstReg, OpRC);
9159 }
9160
9161 return;
9162 }
9163 }
9164
9165 // If this is a v2s copy between 16bit and 32bit reg,
9166 // replace vgpr copy to reg_sequence/extract_subreg
9167 // This can be remove after we have sgpr16 in place
9168 if (ST.useRealTrue16Insts() && Inst.isCopy() &&
9169 Inst.getOperand(1).getReg().isVirtual() &&
9170 RI.isVGPR(MRI, Inst.getOperand(1).getReg())) {
9171 const TargetRegisterClass *SrcRegRC = getOpRegClass(Inst, 1);
9172 if (RI.getMatchingSuperRegClass(NewDstRC, SrcRegRC, AMDGPU::lo16)) {
9173 Register NewDstReg = buildVGPR32FromLo16(
9174 *Inst.getParent(), Inst, Inst.getDebugLoc(),
9175 Inst.getOperand(1).getReg(), AMDGPU::NoSubRegister);
9176 Inst.eraseFromParent();
9177 MRI.replaceRegWith(DstReg, NewDstReg);
9178 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
9179 return;
9180 } else if (RI.getMatchingSuperRegClass(SrcRegRC, NewDstRC,
9181 AMDGPU::lo16)) {
9182 Inst.getOperand(1).setSubReg(AMDGPU::lo16);
9183 Register NewDstReg = MRI.createVirtualRegister(NewDstRC);
9184 MRI.replaceRegWith(DstReg, NewDstReg);
9185 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
9186 return;
9187 }
9188 }
9189
9190 Register NewDstReg = MRI.createVirtualRegister(NewDstRC);
9191 MRI.replaceRegWith(DstReg, NewDstReg);
9192 legalizeOperands(Inst, MDT);
9193 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
9194 return;
9195 }
9196
9197 // Use the new VALU Opcode.
9198 auto NewInstr = BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode))
9199 .setMIFlags(Inst.getFlags());
9200 if (isVOP3(NewOpcode) && !isVOP3(Opcode)) {
9201 // Intersperse VOP3 modifiers among the SALU operands.
9202 NewInstr->addOperand(Inst.getOperand(0));
9203 if (AMDGPU::getNamedOperandIdx(NewOpcode,
9204 AMDGPU::OpName::src0_modifiers) >= 0)
9205 NewInstr.addImm(0);
9206 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0)) {
9207 const MachineOperand &Src = Inst.getOperand(1);
9208 NewInstr->addOperand(Src);
9209 }
9210
9211 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
9212 // We are converting these to a BFE, so we need to add the missing
9213 // operands for the size and offset.
9214 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
9215 NewInstr.addImm(0);
9216 NewInstr.addImm(Size);
9217 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
9218 // The VALU version adds the second operand to the result, so insert an
9219 // extra 0 operand.
9220 NewInstr.addImm(0);
9221 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
9222 const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
9223 // If we need to move this to VGPRs, we need to unpack the second
9224 // operand back into the 2 separate ones for bit offset and width.
9225 assert(OffsetWidthOp.isImm() &&
9226 "Scalar BFE is only implemented for constant width and offset");
9227 uint32_t Imm = OffsetWidthOp.getImm();
9228
9229 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
9230 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
9231 NewInstr.addImm(Offset);
9232 NewInstr.addImm(BitWidth);
9233 } else {
9234 if (AMDGPU::getNamedOperandIdx(NewOpcode,
9235 AMDGPU::OpName::src1_modifiers) >= 0)
9236 NewInstr.addImm(0);
9237 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0)
9238 NewInstr->addOperand(Inst.getOperand(2));
9239 if (AMDGPU::getNamedOperandIdx(NewOpcode,
9240 AMDGPU::OpName::src2_modifiers) >= 0)
9241 NewInstr.addImm(0);
9242 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0)
9243 NewInstr->addOperand(Inst.getOperand(3));
9244 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0)
9245 NewInstr.addImm(0);
9246 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0)
9247 NewInstr.addImm(0);
9248 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0)
9249 NewInstr.addImm(0);
9250 }
9251 } else {
9252 // Just copy the SALU operands.
9253 for (const MachineOperand &Op : Inst.explicit_operands())
9254 NewInstr->addOperand(Op);
9255 }
9256
9257 // Remove any references to SCC. Vector instructions can't read from it, and
9258 // We're just about to add the implicit use / defs of VCC, and we don't want
9259 // both.
9260 bool DeadSCCDef = false;
9261 for (MachineOperand &Op : Inst.implicit_operands()) {
9262 if (Op.getReg() == AMDGPU::SCC) {
9263 // Only propagate through live-def of SCC.
9264 if (Op.isDef()) {
9265 if (Op.isDead())
9266 DeadSCCDef = true;
9267 else
9268 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
9269 continue;
9270 }
9271
9272 addSCCDefsToVALUWorklist(NewInstr, Worklist);
9273 }
9274 }
9275 Inst.eraseFromParent();
9276 Register NewDstReg;
9277 if (NewInstr->getOperand(0).isReg() && NewInstr->getOperand(0).isDef()) {
9278 Register DstReg = NewInstr->getOperand(0).getReg();
9279 assert(DstReg.isVirtual());
9280 // Update the destination register class.
9281 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(*NewInstr);
9282 assert(NewDstRC);
9283 NewDstReg = MRI.createVirtualRegister(NewDstRC);
9284 MRI.replaceRegWith(DstReg, NewDstReg);
9285 }
9286 fixImplicitOperands(*NewInstr);
9287
9288 if (DeadSCCDef) {
9289 // A scalar op with a dead SCC def lowers to a VALU op whose VCC def will
9290 // also be dead.
9291 if (MachineOperand *VCCDef =
9292 NewInstr->findRegisterDefOperand(RI.getVCC(), &RI))
9293 VCCDef->setIsDead();
9294 }
9295
9296 // Legalize the operands
9297 legalizeOperands(*NewInstr, MDT);
9298 if (NewDstReg)
9299 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
9300}
9301
9302// Add/sub require special handling to deal with carry outs.
9303std::pair<bool, MachineBasicBlock *>
9304SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
9305 MachineDominatorTree *MDT) const {
9306 if (ST.hasAddNoCarryInsts()) {
9307 // Assume there is no user of scc since we don't select this in that case.
9308 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
9309 // is used.
9310
9311 MachineBasicBlock &MBB = *Inst.getParent();
9312 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9313
9314 Register OldDstReg = Inst.getOperand(0).getReg();
9315 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9316
9317 unsigned Opc = Inst.getOpcode();
9318 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
9319
9320 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
9321 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
9322
9323 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
9324 Inst.removeOperand(3);
9325
9326 Inst.setDesc(get(NewOpc));
9327 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
9328 Inst.addImplicitDefUseOperands(*MBB.getParent());
9329 MRI.replaceRegWith(OldDstReg, ResultReg);
9330 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT);
9331
9332 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
9333 return std::pair(true, NewBB);
9334 }
9335
9336 return std::pair(false, nullptr);
9337}
9338
9339void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
9340 MachineDominatorTree *MDT) const {
9341
9342 MachineBasicBlock &MBB = *Inst.getParent();
9343 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9344 MachineBasicBlock::iterator MII = Inst;
9345 const DebugLoc &DL = Inst.getDebugLoc();
9346
9347 MachineOperand &Dest = Inst.getOperand(0);
9348 MachineOperand &Src0 = Inst.getOperand(1);
9349 MachineOperand &Src1 = Inst.getOperand(2);
9350 MachineOperand &Cond = Inst.getOperand(3);
9351
9352 Register CondReg = Cond.getReg();
9353 bool IsSCC = (CondReg == AMDGPU::SCC);
9354
9355 // Remove S_CSELECT instructions that we previously inserted to feed the SCC
9356 // condition output from S_CMP into the SGPR condition input of V_CNDMASK. If
9357 // the S_CMP has been promoted to V_CMP then we can feed its SGPR condition
9358 // output directly into the V_CNDMASK.
9359 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
9360 (Src1.getImm() == 0)) {
9361 for (MachineOperand &UseMO :
9363 MachineInstr &UseMI = *UseMO.getParent();
9364 switch (UseMI.getOpcode()) {
9365 case AMDGPU::V_CNDMASK_B16_fake16_e32:
9366 case AMDGPU::V_CNDMASK_B16_fake16_e64:
9367 case AMDGPU::V_CNDMASK_B16_t16_e32:
9368 case AMDGPU::V_CNDMASK_B16_t16_e64:
9369 case AMDGPU::V_CNDMASK_B32_e32:
9370 case AMDGPU::V_CNDMASK_B32_e64:
9371 case AMDGPU::V_CNDMASK_B64_PSEUDO:
9372 if (UseMO.isImplicit() ||
9373 &UseMO == getNamedOperand(UseMI, AMDGPU::OpName::src2))
9374 UseMO.setReg(CondReg);
9375 }
9376 }
9377 if (MRI.use_nodbg_empty(Dest.getReg()))
9378 return;
9379 }
9380
9381 Register NewCondReg = CondReg;
9382 if (IsSCC) {
9383 const TargetRegisterClass *TC = RI.getWaveMaskRegClass();
9384 NewCondReg = MRI.createVirtualRegister(TC);
9385
9386 // Now look for the closest SCC def if it is a copy
9387 // replacing the CondReg with the COPY source register
9388 bool CopyFound = false;
9389 for (MachineInstr &CandI :
9391 Inst.getParent()->rend())) {
9392 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI, false, false) !=
9393 -1) {
9394 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) {
9395 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), NewCondReg)
9396 .addReg(CandI.getOperand(1).getReg());
9397 CopyFound = true;
9398 }
9399 break;
9400 }
9401 }
9402 if (!CopyFound) {
9403 // SCC def is not a copy
9404 // Insert a trivial select instead of creating a copy, because a copy from
9405 // SCC would semantically mean just copying a single bit, but we may need
9406 // the result to be a vector condition mask that needs preserving.
9407 unsigned Opcode =
9408 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
9409 auto NewSelect =
9410 BuildMI(MBB, MII, DL, get(Opcode), NewCondReg).addImm(-1).addImm(0);
9411 NewSelect->getOperand(3).setIsUndef(Cond.isUndef());
9412 }
9413 }
9414
9415 Register NewDestReg = MRI.createVirtualRegister(
9416 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest.getReg())));
9417 MachineInstr *NewInst;
9418 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
9419 NewInst = BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg)
9420 .addImm(0)
9421 .add(Src1) // False
9422 .addImm(0)
9423 .add(Src0) // True
9424 .addReg(NewCondReg);
9425 } else {
9426 NewInst =
9427 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B64_PSEUDO), NewDestReg)
9428 .add(Src1) // False
9429 .add(Src0) // True
9430 .addReg(NewCondReg);
9431 }
9432 MRI.replaceRegWith(Dest.getReg(), NewDestReg);
9433 legalizeOperands(*NewInst, MDT);
9434 addUsersToMoveToVALUWorklist(NewDestReg, MRI, Worklist);
9435}
9436
9437void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
9438 MachineInstr &Inst) const {
9439 MachineBasicBlock &MBB = *Inst.getParent();
9440 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9441 MachineBasicBlock::iterator MII = Inst;
9442 const DebugLoc &DL = Inst.getDebugLoc();
9443
9444 MachineOperand &Dest = Inst.getOperand(0);
9445 MachineOperand &Src = Inst.getOperand(1);
9446 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9447 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9448
9449 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9450 unsigned SubOp =
9451 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9452
9453 MachineInstrBuilder Sub =
9454 BuildMI(MBB, MII, DL, get(SubOp), TmpReg).addImm(0).addReg(Src.getReg());
9455 if (HasCarryOut)
9456 Sub.setOperandDead(3); // Dead vcc
9457
9458 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
9459 .addReg(Src.getReg())
9460 .addReg(TmpReg);
9461
9462 MRI.replaceRegWith(Dest.getReg(), ResultReg);
9463 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
9464}
9465
9466void SIInstrInfo::lowerScalarAbsDiff(SIInstrWorklist &Worklist,
9467 MachineInstr &Inst) const {
9468 MachineBasicBlock &MBB = *Inst.getParent();
9469 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9470 MachineBasicBlock::iterator MII = Inst;
9471 const DebugLoc &DL = Inst.getDebugLoc();
9472
9473 MachineOperand &Dest = Inst.getOperand(0);
9474 MachineOperand &Src1 = Inst.getOperand(1);
9475 MachineOperand &Src2 = Inst.getOperand(2);
9476 Register SubResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9477 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9478 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9479
9480 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9481 unsigned SubOp =
9482 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9483
9484 MachineInstrBuilder Sub1 = BuildMI(MBB, MII, DL, get(SubOp), SubResultReg)
9485 .addReg(Src1.getReg())
9486 .addReg(Src2.getReg());
9487
9488 MachineInstrBuilder Sub2 =
9489 BuildMI(MBB, MII, DL, get(SubOp), TmpReg).addImm(0).addReg(SubResultReg);
9490
9491 if (HasCarryOut) {
9492 Sub1.setOperandDead(3); // Dead vcc
9493 Sub2.setOperandDead(3); // Dead vcc
9494 }
9495
9496 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
9497 .addReg(SubResultReg)
9498 .addReg(TmpReg);
9499
9500 MRI.replaceRegWith(Dest.getReg(), ResultReg);
9501 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
9502}
9503
9504void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
9505 MachineInstr &Inst) const {
9506 MachineBasicBlock &MBB = *Inst.getParent();
9507 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9508 MachineBasicBlock::iterator MII = Inst;
9509 const DebugLoc &DL = Inst.getDebugLoc();
9510
9511 MachineOperand &Dest = Inst.getOperand(0);
9512 MachineOperand &Src0 = Inst.getOperand(1);
9513 MachineOperand &Src1 = Inst.getOperand(2);
9514
9515 if (ST.hasDLInsts()) {
9516 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9517 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
9518 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
9519
9520 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
9521 .add(Src0)
9522 .add(Src1);
9523
9524 MRI.replaceRegWith(Dest.getReg(), NewDest);
9525 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
9526 } else {
9527 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
9528 // invert either source and then perform the XOR. If either source is a
9529 // scalar register, then we can leave the inversion on the scalar unit to
9530 // achieve a better distribution of scalar and vector instructions.
9531 bool Src0IsSGPR = Src0.isReg() &&
9532 RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
9533 bool Src1IsSGPR = Src1.isReg() &&
9534 RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
9535 MachineInstr *Xor;
9536 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
9537 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
9538
9539 // Build a pair of scalar instructions and add them to the work list.
9540 // The next iteration over the work list will lower these to the vector
9541 // unit as necessary.
9542 if (Src0IsSGPR) {
9543 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
9544 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
9545 .addReg(Temp)
9546 .add(Src1);
9547 } else if (Src1IsSGPR) {
9548 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
9549 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
9550 .add(Src0)
9551 .addReg(Temp);
9552 } else {
9553 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
9554 .add(Src0)
9555 .add(Src1);
9556 MachineInstr *Not =
9557 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
9558 Worklist.insert(Not);
9559 }
9560
9561 MRI.replaceRegWith(Dest.getReg(), NewDest);
9562
9563 Worklist.insert(Xor);
9564
9565 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
9566 }
9567}
9568
9569void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
9570 MachineInstr &Inst,
9571 unsigned Opcode) const {
9572 MachineBasicBlock &MBB = *Inst.getParent();
9573 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9574 MachineBasicBlock::iterator MII = Inst;
9575 const DebugLoc &DL = Inst.getDebugLoc();
9576
9577 MachineOperand &Dest = Inst.getOperand(0);
9578 MachineOperand &Src0 = Inst.getOperand(1);
9579 MachineOperand &Src1 = Inst.getOperand(2);
9580
9581 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
9582 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
9583
9584 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
9585 .add(Src0)
9586 .add(Src1);
9587
9588 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
9589 .addReg(Interm);
9590
9591 Worklist.insert(&Op);
9592 Worklist.insert(&Not);
9593
9594 MRI.replaceRegWith(Dest.getReg(), NewDest);
9595 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
9596}
9597
9598void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
9599 MachineInstr &Inst,
9600 unsigned Opcode) const {
9601 MachineBasicBlock &MBB = *Inst.getParent();
9602 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9603 MachineBasicBlock::iterator MII = Inst;
9604 const DebugLoc &DL = Inst.getDebugLoc();
9605
9606 MachineOperand &Dest = Inst.getOperand(0);
9607 MachineOperand &Src0 = Inst.getOperand(1);
9608 MachineOperand &Src1 = Inst.getOperand(2);
9609
9610 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
9611 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
9612
9613 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
9614 .add(Src1);
9615
9616 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
9617 .add(Src0)
9618 .addReg(Interm);
9619
9620 Worklist.insert(&Not);
9621 Worklist.insert(&Op);
9622
9623 MRI.replaceRegWith(Dest.getReg(), NewDest);
9624 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
9625}
9626
9627void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
9628 MachineInstr &Inst, unsigned Opcode,
9629 bool Swap) const {
9630 MachineBasicBlock &MBB = *Inst.getParent();
9631 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9632
9633 MachineOperand &Dest = Inst.getOperand(0);
9634 MachineOperand &Src0 = Inst.getOperand(1);
9635 const DebugLoc &DL = Inst.getDebugLoc();
9636
9637 MachineBasicBlock::iterator MII = Inst;
9638
9639 const MCInstrDesc &InstDesc = get(Opcode);
9640 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9641 MRI.getRegClass(Src0.getReg()) :
9642 &AMDGPU::SGPR_32RegClass;
9643
9644 const TargetRegisterClass *Src0SubRC =
9645 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9646
9647 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
9648 AMDGPU::sub0, Src0SubRC);
9649
9650 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
9651 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
9652 const TargetRegisterClass *NewDestSubRC =
9653 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9654
9655 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
9656 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
9657
9658 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
9659 AMDGPU::sub1, Src0SubRC);
9660
9661 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
9662 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
9663
9664 if (Swap)
9665 std::swap(DestSub0, DestSub1);
9666
9667 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
9668 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
9669 .addReg(DestSub0)
9670 .addImm(AMDGPU::sub0)
9671 .addReg(DestSub1)
9672 .addImm(AMDGPU::sub1);
9673
9674 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
9675
9676 Worklist.insert(&LoHalf);
9677 Worklist.insert(&HiHalf);
9678
9679 // We don't need to legalizeOperands here because for a single operand, src0
9680 // will support any kind of input.
9681
9682 // Move all users of this moved value.
9683 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
9684}
9685
9686// There is not a vector equivalent of s_mul_u64. For this reason, we need to
9687// split the s_mul_u64 in 32-bit vector multiplications.
9688void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
9689 MachineInstr &Inst,
9690 MachineDominatorTree *MDT) const {
9691 MachineBasicBlock &MBB = *Inst.getParent();
9692 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9693
9694 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
9695 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9696 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9697
9698 MachineOperand &Dest = Inst.getOperand(0);
9699 MachineOperand &Src0 = Inst.getOperand(1);
9700 MachineOperand &Src1 = Inst.getOperand(2);
9701 const DebugLoc &DL = Inst.getDebugLoc();
9702 MachineBasicBlock::iterator MII = Inst;
9703
9704 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
9705 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
9706 const TargetRegisterClass *Src0SubRC =
9707 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9708 if (RI.isSGPRClass(Src0SubRC))
9709 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);
9710 const TargetRegisterClass *Src1SubRC =
9711 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9712 if (RI.isSGPRClass(Src1SubRC))
9713 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);
9714
9715 // First, we extract the low 32-bit and high 32-bit values from each of the
9716 // operands.
9717 MachineOperand Op0L =
9718 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
9719 MachineOperand Op1L =
9720 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
9721 MachineOperand Op0H =
9722 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
9723 MachineOperand Op1H =
9724 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
9725
9726 // The multilication is done as follows:
9727 //
9728 // Op1H Op1L
9729 // * Op0H Op0L
9730 // --------------------
9731 // Op1H*Op0L Op1L*Op0L
9732 // + Op1H*Op0H Op1L*Op0H
9733 // -----------------------------------------
9734 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
9735 //
9736 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
9737 // value and that would overflow.
9738 // The low 32-bit value is Op1L*Op0L.
9739 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
9740
9741 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9742 MachineInstr *Op1L_Op0H =
9743 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1L_Op0H_Reg)
9744 .add(Op1L)
9745 .add(Op0H);
9746
9747 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9748 MachineInstr *Op1H_Op0L =
9749 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1H_Op0L_Reg)
9750 .add(Op1H)
9751 .add(Op0L);
9752
9753 Register CarryReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9754 MachineInstr *Carry =
9755 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_HI_U32_e64), CarryReg)
9756 .add(Op1L)
9757 .add(Op0L);
9758
9759 MachineInstr *LoHalf =
9760 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)
9761 .add(Op1L)
9762 .add(Op0L);
9763
9764 Register AddReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9765 MachineInstr *Add = BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), AddReg)
9766 .addReg(Op1L_Op0H_Reg)
9767 .addReg(Op1H_Op0L_Reg);
9768
9769 MachineInstr *HiHalf =
9770 BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), DestSub1)
9771 .addReg(AddReg)
9772 .addReg(CarryReg);
9773
9774 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
9775 .addReg(DestSub0)
9776 .addImm(AMDGPU::sub0)
9777 .addReg(DestSub1)
9778 .addImm(AMDGPU::sub1);
9779
9780 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
9781
9782 // Try to legalize the operands in case we need to swap the order to keep it
9783 // valid.
9784 legalizeOperands(*Op1L_Op0H, MDT);
9785 legalizeOperands(*Op1H_Op0L, MDT);
9786 legalizeOperands(*Carry, MDT);
9787 legalizeOperands(*LoHalf, MDT);
9788 legalizeOperands(*Add, MDT);
9789 legalizeOperands(*HiHalf, MDT);
9790
9791 // Move all users of this moved value.
9792 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
9793}
9794
9795// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
9796// multiplications.
9797void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
9798 MachineInstr &Inst,
9799 MachineDominatorTree *MDT) const {
9800 MachineBasicBlock &MBB = *Inst.getParent();
9801 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9802
9803 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
9804 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9805 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9806
9807 MachineOperand &Dest = Inst.getOperand(0);
9808 MachineOperand &Src0 = Inst.getOperand(1);
9809 MachineOperand &Src1 = Inst.getOperand(2);
9810 const DebugLoc &DL = Inst.getDebugLoc();
9811 MachineBasicBlock::iterator MII = Inst;
9812
9813 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
9814 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
9815 const TargetRegisterClass *Src0SubRC =
9816 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9817 if (RI.isSGPRClass(Src0SubRC))
9818 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);
9819 const TargetRegisterClass *Src1SubRC =
9820 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9821 if (RI.isSGPRClass(Src1SubRC))
9822 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);
9823
9824 // First, we extract the low 32-bit and high 32-bit values from each of the
9825 // operands.
9826 MachineOperand Op0L =
9827 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
9828 MachineOperand Op1L =
9829 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
9830
9831 unsigned Opc = Inst.getOpcode();
9832 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
9833 ? AMDGPU::V_MUL_HI_U32_e64
9834 : AMDGPU::V_MUL_HI_I32_e64;
9835 MachineInstr *HiHalf =
9836 BuildMI(MBB, MII, DL, get(NewOpc), DestSub1).add(Op1L).add(Op0L);
9837
9838 MachineInstr *LoHalf =
9839 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)
9840 .add(Op1L)
9841 .add(Op0L);
9842
9843 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
9844 .addReg(DestSub0)
9845 .addImm(AMDGPU::sub0)
9846 .addReg(DestSub1)
9847 .addImm(AMDGPU::sub1);
9848
9849 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
9850
9851 // Try to legalize the operands in case we need to swap the order to keep it
9852 // valid.
9853 legalizeOperands(*HiHalf, MDT);
9854 legalizeOperands(*LoHalf, MDT);
9855
9856 // Move all users of this moved value.
9857 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
9858}
9859
9860void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
9861 MachineInstr &Inst, unsigned Opcode,
9862 MachineDominatorTree *MDT) const {
9863 MachineBasicBlock &MBB = *Inst.getParent();
9864 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9865
9866 MachineOperand &Dest = Inst.getOperand(0);
9867 MachineOperand &Src0 = Inst.getOperand(1);
9868 MachineOperand &Src1 = Inst.getOperand(2);
9869 const DebugLoc &DL = Inst.getDebugLoc();
9870
9871 MachineBasicBlock::iterator MII = Inst;
9872
9873 const MCInstrDesc &InstDesc = get(Opcode);
9874 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9875 MRI.getRegClass(Src0.getReg()) :
9876 &AMDGPU::SGPR_32RegClass;
9877
9878 const TargetRegisterClass *Src0SubRC =
9879 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9880 const TargetRegisterClass *Src1RC = Src1.isReg() ?
9881 MRI.getRegClass(Src1.getReg()) :
9882 &AMDGPU::SGPR_32RegClass;
9883
9884 const TargetRegisterClass *Src1SubRC =
9885 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9886
9887 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
9888 AMDGPU::sub0, Src0SubRC);
9889 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
9890 AMDGPU::sub0, Src1SubRC);
9891 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
9892 AMDGPU::sub1, Src0SubRC);
9893 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
9894 AMDGPU::sub1, Src1SubRC);
9895
9896 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
9897 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
9898 const TargetRegisterClass *NewDestSubRC =
9899 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9900
9901 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
9902 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
9903 .add(SrcReg0Sub0)
9904 .add(SrcReg1Sub0);
9905
9906 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
9907 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
9908 .add(SrcReg0Sub1)
9909 .add(SrcReg1Sub1);
9910
9911 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
9912 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
9913 .addReg(DestSub0)
9914 .addImm(AMDGPU::sub0)
9915 .addReg(DestSub1)
9916 .addImm(AMDGPU::sub1);
9917
9918 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
9919
9920 Worklist.insert(&LoHalf);
9921 Worklist.insert(&HiHalf);
9922
9923 // Move all users of this moved value.
9924 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
9925}
9926
9927void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
9928 MachineInstr &Inst,
9929 MachineDominatorTree *MDT) const {
9930 MachineBasicBlock &MBB = *Inst.getParent();
9931 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9932
9933 MachineOperand &Dest = Inst.getOperand(0);
9934 MachineOperand &Src0 = Inst.getOperand(1);
9935 MachineOperand &Src1 = Inst.getOperand(2);
9936 const DebugLoc &DL = Inst.getDebugLoc();
9937
9938 MachineBasicBlock::iterator MII = Inst;
9939
9940 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
9941
9942 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
9943
9944 MachineOperand* Op0;
9945 MachineOperand* Op1;
9946
9947 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) {
9948 Op0 = &Src0;
9949 Op1 = &Src1;
9950 } else {
9951 Op0 = &Src1;
9952 Op1 = &Src0;
9953 }
9954
9955 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)
9956 .add(*Op0);
9957
9958 Register NewDest = MRI.createVirtualRegister(DestRC);
9959
9960 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)
9961 .addReg(Interm)
9962 .add(*Op1);
9963
9964 MRI.replaceRegWith(Dest.getReg(), NewDest);
9965
9966 Worklist.insert(&Xor);
9967}
9968
9969void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
9970 MachineInstr &Inst) const {
9971 MachineBasicBlock &MBB = *Inst.getParent();
9972 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9973
9974 MachineBasicBlock::iterator MII = Inst;
9975 const DebugLoc &DL = Inst.getDebugLoc();
9976
9977 MachineOperand &Dest = Inst.getOperand(0);
9978 MachineOperand &Src = Inst.getOperand(1);
9979
9980 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
9981 const TargetRegisterClass *SrcRC = Src.isReg() ?
9982 MRI.getRegClass(Src.getReg()) :
9983 &AMDGPU::SGPR_32RegClass;
9984
9985 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9986 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9987
9988 const TargetRegisterClass *SrcSubRC =
9989 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9990
9991 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
9992 AMDGPU::sub0, SrcSubRC);
9993 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
9994 AMDGPU::sub1, SrcSubRC);
9995
9996 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
9997
9998 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
9999
10000 MRI.replaceRegWith(Dest.getReg(), ResultReg);
10001
10002 // We don't need to legalize operands here. src0 for either instruction can be
10003 // an SGPR, and the second input is unused or determined here.
10004 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
10005}
10006
10007void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
10008 MachineInstr &Inst) const {
10009 MachineBasicBlock &MBB = *Inst.getParent();
10010 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10011 MachineBasicBlock::iterator MII = Inst;
10012 const DebugLoc &DL = Inst.getDebugLoc();
10013
10014 MachineOperand &Dest = Inst.getOperand(0);
10015 uint32_t Imm = Inst.getOperand(2).getImm();
10016 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
10017 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
10018
10019 (void) Offset;
10020
10021 // Only sext_inreg cases handled.
10022 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
10023 Offset == 0 && "Not implemented");
10024
10025 if (BitWidth < 32) {
10026 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10027 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10028 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
10029
10030 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo)
10031 .addReg(Inst.getOperand(1).getReg(), {}, AMDGPU::sub0)
10032 .addImm(0)
10033 .addImm(BitWidth);
10034
10035 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
10036 .addImm(31)
10037 .addReg(MidRegLo);
10038
10039 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
10040 .addReg(MidRegLo)
10041 .addImm(AMDGPU::sub0)
10042 .addReg(MidRegHi)
10043 .addImm(AMDGPU::sub1);
10044
10045 MRI.replaceRegWith(Dest.getReg(), ResultReg);
10046 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
10047 return;
10048 }
10049
10050 MachineOperand &Src = Inst.getOperand(1);
10051 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10052 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
10053
10054 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
10055 .addImm(31)
10056 .addReg(Src.getReg(), {}, AMDGPU::sub0);
10057
10058 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
10059 .addReg(Src.getReg(), {}, AMDGPU::sub0)
10060 .addImm(AMDGPU::sub0)
10061 .addReg(TmpReg)
10062 .addImm(AMDGPU::sub1);
10063
10064 MRI.replaceRegWith(Dest.getReg(), ResultReg);
10065 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
10066}
10067
10068void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
10069 MachineInstr &Inst, unsigned Opcode,
10070 MachineDominatorTree *MDT) const {
10071 // (S_FLBIT_I32_B64 hi:lo) ->
10072 // -> (umin (V_FFBH_U32_e32 hi), (or (V_FFBH_U32_e32 lo), 32))
10073 // (S_FF1_I32_B64 hi:lo) ->
10074 // ->(umin (or (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
10075
10076 MachineBasicBlock &MBB = *Inst.getParent();
10077 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10078 MachineBasicBlock::iterator MII = Inst;
10079 const DebugLoc &DL = Inst.getDebugLoc();
10080
10081 MachineOperand &Dest = Inst.getOperand(0);
10082 MachineOperand &Src = Inst.getOperand(1);
10083
10084 const MCInstrDesc &InstDesc = get(Opcode);
10085
10086 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
10087
10088 const TargetRegisterClass *SrcRC =
10089 Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass;
10090 const TargetRegisterClass *SrcSubRC =
10091 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
10092
10093 MachineOperand SrcRegSub0 =
10094 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC);
10095 MachineOperand SrcRegSub1 =
10096 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC);
10097
10098 Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10099 Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10100 Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10101 Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10102
10103 BuildMI(MBB, MII, DL, InstDesc, MidReg1).add(SrcRegSub0);
10104
10105 BuildMI(MBB, MII, DL, InstDesc, MidReg2).add(SrcRegSub1);
10106
10107 BuildMI(MBB, MII, DL, get(AMDGPU::V_OR_B32_e32), MidReg3)
10108 .addImm(32)
10109 .addReg(IsCtlz ? MidReg1 : MidReg2);
10110
10111 BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4)
10112 .addReg(MidReg3)
10113 .addReg(IsCtlz ? MidReg2 : MidReg1);
10114
10115 MRI.replaceRegWith(Dest.getReg(), MidReg4);
10116
10117 addUsersToMoveToVALUWorklist(MidReg4, MRI, Worklist);
10118}
10119
10120void SIInstrInfo::addUsersToMoveToVALUWorklist(
10122 SIInstrWorklist &Worklist) const {
10123 for (MachineOperand &MO : make_early_inc_range(MRI.use_operands(DstReg))) {
10124 MachineInstr &UseMI = *MO.getParent();
10125
10126 unsigned OpNo = 0;
10127
10128 switch (UseMI.getOpcode()) {
10129 case AMDGPU::COPY:
10130 case AMDGPU::WQM:
10131 case AMDGPU::SOFT_WQM:
10132 case AMDGPU::STRICT_WWM:
10133 case AMDGPU::STRICT_WQM:
10134 case AMDGPU::REG_SEQUENCE:
10135 case AMDGPU::PHI:
10136 case AMDGPU::INSERT_SUBREG:
10137 break;
10138 default:
10139 OpNo = MO.getOperandNo();
10140 break;
10141 }
10142
10143 const TargetRegisterClass *OpRC = getOpRegClass(UseMI, OpNo);
10144 MRI.constrainRegClass(DstReg, OpRC);
10145
10146 if (!RI.hasVectorRegisters(OpRC))
10147 Worklist.insert(&UseMI);
10148 else
10149 // Legalization could change user list.
10151 }
10152}
10153
10154void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
10156 MachineInstr &Inst) const {
10157 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10158 MachineBasicBlock *MBB = Inst.getParent();
10159 MachineOperand &Src0 = Inst.getOperand(1);
10160 MachineOperand &Src1 = Inst.getOperand(2);
10161 const DebugLoc &DL = Inst.getDebugLoc();
10162
10163 if (ST.useRealTrue16Insts()) {
10164 Register SrcReg0, SrcReg1;
10165 if (!Src0.isReg() || !RI.isVGPR(MRI, Src0.getReg())) {
10166 SrcReg0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10167 BuildMI(*MBB, Inst, DL,
10168 get(Src0.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), SrcReg0)
10169 .add(Src0);
10170 } else {
10171 SrcReg0 = Src0.getReg();
10172 }
10173
10174 if (!Src1.isReg() || !RI.isVGPR(MRI, Src1.getReg())) {
10175 SrcReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10176 BuildMI(*MBB, Inst, DL,
10177 get(Src1.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), SrcReg1)
10178 .add(Src1);
10179 } else {
10180 SrcReg1 = Src1.getReg();
10181 }
10182
10183 bool isSrc0Reg16 = MRI.constrainRegClass(SrcReg0, &AMDGPU::VGPR_16RegClass);
10184 bool isSrc1Reg16 = MRI.constrainRegClass(SrcReg1, &AMDGPU::VGPR_16RegClass);
10185
10186 auto NewMI = BuildMI(*MBB, Inst, DL, get(AMDGPU::REG_SEQUENCE), ResultReg);
10187 switch (Inst.getOpcode()) {
10188 case AMDGPU::S_PACK_LL_B32_B16:
10189 NewMI
10190 .addReg(SrcReg0, {},
10191 isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10192 .addImm(AMDGPU::lo16)
10193 .addReg(SrcReg1, {},
10194 isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10195 .addImm(AMDGPU::hi16);
10196 break;
10197 case AMDGPU::S_PACK_LH_B32_B16:
10198 NewMI
10199 .addReg(SrcReg0, {},
10200 isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10201 .addImm(AMDGPU::lo16)
10202 .addReg(SrcReg1, {}, AMDGPU::hi16)
10203 .addImm(AMDGPU::hi16);
10204 break;
10205 case AMDGPU::S_PACK_HL_B32_B16:
10206 NewMI.addReg(SrcReg0, {}, AMDGPU::hi16)
10207 .addImm(AMDGPU::lo16)
10208 .addReg(SrcReg1, {},
10209 isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10210 .addImm(AMDGPU::hi16);
10211 break;
10212 case AMDGPU::S_PACK_HH_B32_B16:
10213 NewMI.addReg(SrcReg0, {}, AMDGPU::hi16)
10214 .addImm(AMDGPU::lo16)
10215 .addReg(SrcReg1, {}, AMDGPU::hi16)
10216 .addImm(AMDGPU::hi16);
10217 break;
10218 default:
10219 llvm_unreachable("unhandled s_pack_* instruction");
10220 }
10221
10222 MachineOperand &Dest = Inst.getOperand(0);
10223 MRI.replaceRegWith(Dest.getReg(), ResultReg);
10224 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
10225 return;
10226 }
10227
10228 switch (Inst.getOpcode()) {
10229 case AMDGPU::S_PACK_LL_B32_B16: {
10230 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10231 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10232
10233 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
10234 // 0.
10235 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
10236 .addImm(0xffff);
10237
10238 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
10239 .addReg(ImmReg)
10240 .add(Src0);
10241
10242 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)
10243 .add(Src1)
10244 .addImm(16)
10245 .addReg(TmpReg);
10246 break;
10247 }
10248 case AMDGPU::S_PACK_LH_B32_B16: {
10249 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10250 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
10251 .addImm(0xffff);
10252 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg)
10253 .addReg(ImmReg)
10254 .add(Src0)
10255 .add(Src1);
10256 break;
10257 }
10258 case AMDGPU::S_PACK_HL_B32_B16: {
10259 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10260 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
10261 .addImm(16)
10262 .add(Src0);
10263 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)
10264 .add(Src1)
10265 .addImm(16)
10266 .addReg(TmpReg);
10267 break;
10268 }
10269 case AMDGPU::S_PACK_HH_B32_B16: {
10270 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10271 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
10272 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
10273 .addImm(16)
10274 .add(Src0);
10275 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
10276 .addImm(0xffff0000);
10277 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg)
10278 .add(Src1)
10279 .addReg(ImmReg)
10280 .addReg(TmpReg);
10281 break;
10282 }
10283 default:
10284 llvm_unreachable("unhandled s_pack_* instruction");
10285 }
10286
10287 MachineOperand &Dest = Inst.getOperand(0);
10288 MRI.replaceRegWith(Dest.getReg(), ResultReg);
10289 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
10290}
10291
10292void SIInstrInfo::addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
10293 MachineInstr &SCCDefInst,
10294 SIInstrWorklist &Worklist,
10295 Register NewCond) const {
10296
10297 // Ensure that def inst defines SCC, which is still live.
10298 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
10299 !Op.isDead() && Op.getParent() == &SCCDefInst);
10300 SmallVector<MachineInstr *, 4> CopyToDelete;
10301 // This assumes that all the users of SCC are in the same block
10302 // as the SCC def.
10303 for (MachineInstr &MI : // Skip the def inst itself.
10304 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)),
10305 SCCDefInst.getParent()->end())) {
10306 // Check if SCC is used first.
10307 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, &RI, false);
10308 if (SCCIdx != -1) {
10309 if (MI.isCopy()) {
10310 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10311 Register DestReg = MI.getOperand(0).getReg();
10312
10313 MRI.replaceRegWith(DestReg, NewCond);
10314 CopyToDelete.push_back(&MI);
10315 } else {
10316
10317 if (NewCond.isValid())
10318 MI.getOperand(SCCIdx).setReg(NewCond);
10319
10320 Worklist.insert(&MI);
10321 }
10322 }
10323 // Exit if we find another SCC def.
10324 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI, false, false) != -1)
10325 break;
10326 }
10327 for (auto &Copy : CopyToDelete)
10328 Copy->eraseFromParent();
10329}
10330
10331// Instructions that use SCC may be converted to VALU instructions. When that
10332// happens, the SCC register is changed to VCC_LO. The instruction that defines
10333// SCC must be changed to an instruction that defines VCC. This function makes
10334// sure that the instruction that defines SCC is added to the moveToVALU
10335// worklist.
10336void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
10337 SIInstrWorklist &Worklist) const {
10338 // Look for a preceding instruction that either defines VCC or SCC. If VCC
10339 // then there is nothing to do because the defining instruction has been
10340 // converted to a VALU already. If SCC then that instruction needs to be
10341 // converted to a VALU.
10342 for (MachineInstr &MI :
10343 make_range(std::next(MachineBasicBlock::reverse_iterator(SCCUseInst)),
10344 SCCUseInst->getParent()->rend())) {
10345 if (MI.modifiesRegister(AMDGPU::VCC, &RI))
10346 break;
10347 if (MI.definesRegister(AMDGPU::SCC, &RI)) {
10348 Worklist.insert(&MI);
10349 break;
10350 }
10351 }
10352}
10353
10354const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
10355 const MachineInstr &Inst) const {
10356 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
10357
10358 switch (Inst.getOpcode()) {
10359 // For target instructions, getOpRegClass just returns the virtual register
10360 // class associated with the operand, so we need to find an equivalent VGPR
10361 // register class in order to move the instruction to the VALU.
10362 case AMDGPU::COPY:
10363 case AMDGPU::PHI:
10364 case AMDGPU::REG_SEQUENCE:
10365 case AMDGPU::INSERT_SUBREG:
10366 case AMDGPU::WQM:
10367 case AMDGPU::SOFT_WQM:
10368 case AMDGPU::STRICT_WWM:
10369 case AMDGPU::STRICT_WQM: {
10370 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1);
10371 if (RI.isAGPRClass(SrcRC)) {
10372 if (RI.isAGPRClass(NewDstRC))
10373 return nullptr;
10374
10375 switch (Inst.getOpcode()) {
10376 case AMDGPU::PHI:
10377 case AMDGPU::REG_SEQUENCE:
10378 case AMDGPU::INSERT_SUBREG:
10379 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);
10380 break;
10381 default:
10382 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
10383 }
10384
10385 if (!NewDstRC)
10386 return nullptr;
10387 } else {
10388 if (!RI.isSGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
10389 return nullptr;
10390
10391 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
10392 if (!NewDstRC)
10393 return nullptr;
10394 }
10395
10396 return NewDstRC;
10397 }
10398 default:
10399 return NewDstRC;
10400 }
10401}
10402
10403// Find the one SGPR operand we are allowed to use.
10404Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
10405 int OpIndices[3]) const {
10406 const MCInstrDesc &Desc = MI.getDesc();
10407
10408 // Find the one SGPR operand we are allowed to use.
10409 //
10410 // First we need to consider the instruction's operand requirements before
10411 // legalizing. Some operands are required to be SGPRs, such as implicit uses
10412 // of VCC, but we are still bound by the constant bus requirement to only use
10413 // one.
10414 //
10415 // If the operand's class is an SGPR, we can never move it.
10416
10417 Register SGPRReg = findImplicitSGPRRead(MI);
10418 if (SGPRReg)
10419 return SGPRReg;
10420
10421 Register UsedSGPRs[3] = {Register()};
10422 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10423
10424 for (unsigned i = 0; i < 3; ++i) {
10425 int Idx = OpIndices[i];
10426 if (Idx == -1)
10427 break;
10428
10429 const MachineOperand &MO = MI.getOperand(Idx);
10430 if (!MO.isReg())
10431 continue;
10432
10433 // Is this operand statically required to be an SGPR based on the operand
10434 // constraints?
10435 const TargetRegisterClass *OpRC =
10436 RI.getRegClass(getOpRegClassID(Desc.operands()[Idx]));
10437 bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
10438 if (IsRequiredSGPR)
10439 return MO.getReg();
10440
10441 // If this could be a VGPR or an SGPR, Check the dynamic register class.
10442 Register Reg = MO.getReg();
10443 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
10444 if (RI.isSGPRClass(RegRC))
10445 UsedSGPRs[i] = Reg;
10446 }
10447
10448 // We don't have a required SGPR operand, so we have a bit more freedom in
10449 // selecting operands to move.
10450
10451 // Try to select the most used SGPR. If an SGPR is equal to one of the
10452 // others, we choose that.
10453 //
10454 // e.g.
10455 // V_FMA_F32 v0, s0, s0, s0 -> No moves
10456 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
10457
10458 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
10459 // prefer those.
10460
10461 if (UsedSGPRs[0]) {
10462 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
10463 SGPRReg = UsedSGPRs[0];
10464 }
10465
10466 if (!SGPRReg && UsedSGPRs[1]) {
10467 if (UsedSGPRs[1] == UsedSGPRs[2])
10468 SGPRReg = UsedSGPRs[1];
10469 }
10470
10471 return SGPRReg;
10472}
10473
10475 AMDGPU::OpName OperandName) const {
10476 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)
10477 return nullptr;
10478
10479 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
10480 if (Idx == -1)
10481 return nullptr;
10482
10483 return &MI.getOperand(Idx);
10484}
10485
10487 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
10488 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
10491 return (Format << 44) |
10492 (1ULL << 56) | // RESOURCE_LEVEL = 1
10493 (3ULL << 60); // OOB_SELECT = 3
10494 }
10495
10496 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
10497 if (ST.isAmdHsaOS()) {
10498 // Set ATC = 1. GFX9 doesn't have this bit.
10499 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10500 RsrcDataFormat |= (1ULL << 56);
10501
10502 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
10503 // BTW, it disables TC L2 and therefore decreases performance.
10504 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
10505 RsrcDataFormat |= (2ULL << 59);
10506 }
10507
10508 return RsrcDataFormat;
10509}
10510
10512 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
10514 0xffffffff; // Size;
10515
10516 // GFX9 doesn't have ELEMENT_SIZE.
10517 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
10518 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1;
10519 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
10520 }
10521
10522 // IndexStride = 64 / 32.
10523 uint64_t IndexStride = ST.isWave64() ? 3 : 2;
10524 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
10525
10526 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
10527 // Clear them unless we want a huge stride.
10528 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
10529 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
10530 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
10531
10532 return Rsrc23;
10533}
10534
10536 unsigned Opc = MI.getOpcode();
10537
10538 return isSMRD(Opc);
10539}
10540
10542 return get(Opc).mayLoad() &&
10543 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc));
10544}
10545
10547 TypeSize &MemBytes) const {
10548 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
10549 if (!Addr || !Addr->isFI())
10550 return Register();
10551
10552 assert(!MI.memoperands_empty() &&
10553 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
10554
10555 FrameIndex = Addr->getIndex();
10556
10557 int VDataIdx =
10558 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
10559 MemBytes = TypeSize::getFixed(getOpSize(MI.getOpcode(), VDataIdx));
10560 return MI.getOperand(VDataIdx).getReg();
10561}
10562
10564 TypeSize &MemBytes) const {
10565 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
10566 assert(Addr && Addr->isFI());
10567 FrameIndex = Addr->getIndex();
10568
10569 int DataIdx =
10570 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::data);
10571 MemBytes = TypeSize::getFixed(getOpSize(MI.getOpcode(), DataIdx));
10572 return MI.getOperand(DataIdx).getReg();
10573}
10574
10576 int &FrameIndex,
10577 TypeSize &MemBytes) const {
10578 if (!MI.mayLoad())
10579 return Register();
10580
10581 if (isMUBUF(MI) || isVGPRSpill(MI))
10582 return isStackAccess(MI, FrameIndex, MemBytes);
10583
10584 if (isSGPRSpill(MI))
10585 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10586
10587 return Register();
10588}
10589
10591 int &FrameIndex,
10592 TypeSize &MemBytes) const {
10593 if (!MI.mayStore())
10594 return Register();
10595
10596 if (isMUBUF(MI) || isVGPRSpill(MI))
10597 return isStackAccess(MI, FrameIndex, MemBytes);
10598
10599 if (isSGPRSpill(MI))
10600 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10601
10602 return Register();
10603}
10604
10606 unsigned Opc = MI.getOpcode();
10608 unsigned DescSize = Desc.getSize();
10609 if (Opc == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay)
10610 return DescSize + 12; // Three scalar moves around the load.
10611
10612 // If we have a definitive size, we can use it. Otherwise we need to inspect
10613 // the operands to know the size.
10614 if (isFixedSize(MI)) {
10615 unsigned Size = DescSize;
10616
10617 // If we hit the buggy offset, an extra nop will be inserted in MC so
10618 // estimate the worst case.
10619 if (MI.isBranch() && ST.hasOffset3fBug())
10620 Size += 4;
10621
10622 return Size;
10623 }
10624
10625 // Instructions may have a 32-bit literal encoded after them. Check
10626 // operands that could ever be literals.
10627 if (isVALU(MI, /*AllowLDSDMA=*/false) || isSALU(MI)) {
10628 if (isDPP(MI))
10629 return DescSize;
10630 bool HasLiteral = false;
10631 unsigned LiteralSize = 4;
10632 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
10633 const MachineOperand &Op = MI.getOperand(I);
10634 const MCOperandInfo &OpInfo = Desc.operands()[I];
10635 if (!Op.isReg() && !isInlineConstant(Op, OpInfo)) {
10636 HasLiteral = true;
10637 if (ST.has64BitLiterals()) {
10638 switch (OpInfo.OperandType) {
10639 default:
10640 break;
10643 if (!AMDGPU::isValid32BitLiteral(Op.getImm(), true))
10644 LiteralSize = 8;
10645 break;
10648 // A 32-bit literal is only valid when the value fits in BOTH signed
10649 // and unsigned 32-bit ranges [0, 2^31-1], matching the MC code
10650 // emitter's getLit64Encoding logic. This is because of the lack of
10651 // abilility to tell signedness of the literal, therefore we need to
10652 // be conservative and assume values outside this range require a
10653 // 64-bit literal encoding (8 bytes).
10654 if (!Op.isImm() || !isInt<32>(Op.getImm()) ||
10655 !isUInt<32>(Op.getImm()))
10656 LiteralSize = 8;
10657 break;
10658 }
10659 }
10660 break;
10661 }
10662 }
10663 return HasLiteral ? DescSize + LiteralSize : DescSize;
10664 }
10665
10666 // Check whether we have extra NSA words.
10667 if (isMIMG(MI)) {
10668 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
10669 if (VAddr0Idx < 0)
10670 return 8;
10671
10672 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
10673 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
10674 }
10675
10676 switch (Opc) {
10677 case TargetOpcode::BUNDLE:
10678 return getInstBundleSize(MI);
10679 case TargetOpcode::INLINEASM:
10680 case TargetOpcode::INLINEASM_BR: {
10681 const MachineFunction *MF = MI.getMF();
10682 const char *AsmStr = MI.getOperand(0).getSymbolName();
10683 return getInlineAsmLength(AsmStr, MF->getTarget().getMCAsmInfo(), &ST);
10684 }
10685 default:
10686 if (MI.isMetaInstruction())
10687 return 0;
10688
10689 // If D16 Pseudo inst, get correct MC code size
10690 const auto *D16Info = AMDGPU::getT16D16Helper(Opc);
10691 if (D16Info) {
10692 // Assume d16_lo/hi inst are always in same size
10693 unsigned LoInstOpcode = D16Info->LoOp;
10694 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(LoInstOpcode);
10695 DescSize = Desc.getSize();
10696 }
10697
10698 // If FMA Pseudo inst, get correct MC code size
10699 if (Opc == AMDGPU::V_FMA_MIX_F16_t16 || Opc == AMDGPU::V_FMA_MIX_BF16_t16) {
10700 // All potential lowerings are the same size; arbitrarily pick one.
10701 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(AMDGPU::V_FMA_MIXLO_F16);
10702 DescSize = Desc.getSize();
10703 }
10704
10705 return DescSize;
10706 }
10707}
10708
10711 if (MI.isBranch() && ST.hasOffset3fBug())
10712 return InstSizeVerifyMode::NoVerify;
10713 return InstSizeVerifyMode::ExactSize;
10714}
10715
10718 static const std::pair<int, const char *> TargetIndices[] = {
10719 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
10720 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
10721 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
10722 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
10723 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
10724 return ArrayRef(TargetIndices);
10725}
10726
10727/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
10728/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
10731 const ScheduleDAG *DAG) const {
10732 return new GCNHazardRecognizer(DAG->MF);
10733}
10734
10735/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
10736/// pass.
10743
10744// Called during:
10745// - pre-RA scheduling and post-RA scheduling
10748 const ScheduleDAGMI *DAG) const {
10749 // Borrowed from Arm Target
10750 // We would like to restrict this hazard recognizer to only
10751 // post-RA scheduling; we can tell that we're post-RA because we don't
10752 // track VRegLiveness.
10753 if (!DAG->hasVRegLiveness())
10754 return new GCNHazardRecognizer(DAG->MF);
10756}
10757
10758std::pair<unsigned, unsigned>
10760 return std::pair(TF & MO_MASK, TF & ~MO_MASK);
10761}
10762
10765 static const std::pair<unsigned, const char *> TargetFlags[] = {
10766 {MO_GOTPCREL, "amdgpu-gotprel"},
10767 {MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo"},
10768 {MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi"},
10769 {MO_GOTPCREL64, "amdgpu-gotprel64"},
10770 {MO_REL32_LO, "amdgpu-rel32-lo"},
10771 {MO_REL32_HI, "amdgpu-rel32-hi"},
10772 {MO_REL64, "amdgpu-rel64"},
10773 {MO_ABS32_LO, "amdgpu-abs32-lo"},
10774 {MO_ABS32_HI, "amdgpu-abs32-hi"},
10775 {MO_ABS64, "amdgpu-abs64"},
10776 };
10777
10778 return ArrayRef(TargetFlags);
10779}
10780
10783 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10784 {
10785 {MONoClobber, "amdgpu-noclobber"},
10786 {MOLastUse, "amdgpu-last-use"},
10787 {MOCooperative, "amdgpu-cooperative"},
10788 {MOThreadPrivate, "amdgpu-thread-private"},
10789 {MOCFSB0, "amdgpu-cfs0"},
10790 {MOCFSB1, "amdgpu-cfs1"},
10791 };
10792
10793 return ArrayRef(TargetFlags);
10794}
10795
10797 const MachineFunction &MF) const {
10799 assert(SrcReg.isVirtual());
10800 if (MFI->checkFlag(SrcReg, AMDGPU::VirtRegFlag::WWM_REG))
10801 return AMDGPU::WWM_COPY;
10802
10803 return AMDGPU::COPY;
10804}
10805
10807 uint32_t Opcode = MI.getOpcode();
10808 // Check if it is SGPR spill or wwm-register spill Opcode.
10809 if (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode))
10810 return true;
10811
10812 const MachineFunction *MF = MI.getMF();
10813 const MachineRegisterInfo &MRI = MF->getRegInfo();
10815
10816 // See if this is Liverange split instruction inserted for SGPR or
10817 // wwm-register. The implicit def inserted for wwm-registers should also be
10818 // included as they can appear at the bb begin.
10819 bool IsLRSplitInst = MI.getFlag(MachineInstr::LRSplit);
10820 if (!IsLRSplitInst && Opcode != AMDGPU::IMPLICIT_DEF)
10821 return false;
10822
10823 Register Reg = MI.getOperand(0).getReg();
10824 if (RI.isSGPRClass(RI.getRegClassForReg(MRI, Reg)))
10825 return IsLRSplitInst;
10826
10827 return MFI->isWWMReg(Reg);
10828}
10829
10831 Register Reg) const {
10832 // We need to handle instructions which may be inserted during register
10833 // allocation to handle the prolog. The initial prolog instruction may have
10834 // been separated from the start of the block by spills and copies inserted
10835 // needed by the prolog. However, the insertions for scalar registers can
10836 // always be placed at the BB top as they are independent of the exec mask
10837 // value.
10838 bool IsNullOrVectorRegister = true;
10839 if (Reg) {
10840 const MachineFunction *MF = MI.getMF();
10841 const MachineRegisterInfo &MRI = MF->getRegInfo();
10842 IsNullOrVectorRegister = !RI.isSGPRClass(RI.getRegClassForReg(MRI, Reg));
10843 }
10844
10845 return IsNullOrVectorRegister &&
10846 (canAddToBBProlog(MI) ||
10847 (!MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
10848 MI.modifiesRegister(AMDGPU::EXEC, &RI)));
10849}
10850
10854 const DebugLoc &DL,
10855 Register DestReg) const {
10856 if (ST.hasAddNoCarryInsts())
10857 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
10858
10859 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10860 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC());
10861 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC());
10862
10863 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)
10864 .addReg(UnusedCarry, RegState::Define | RegState::Dead);
10865}
10866
10869 const DebugLoc &DL,
10870 Register DestReg,
10871 RegScavenger &RS) const {
10872 if (ST.hasAddNoCarryInsts())
10873 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg);
10874
10875 // If available, prefer to use vcc.
10876 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC)
10877 ? Register(RI.getVCC())
10878 : RS.scavengeRegisterBackwards(
10879 *RI.getBoolRC(), I, /* RestoreAfter */ false,
10880 0, /* AllowSpill */ false);
10881
10882 // TODO: Users need to deal with this.
10883 if (!UnusedCarry.isValid())
10884 return MachineInstrBuilder();
10885
10886 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)
10887 .addReg(UnusedCarry, RegState::Define | RegState::Dead);
10888}
10889
10890bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
10891 switch (Opcode) {
10892 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
10893 case AMDGPU::SI_KILL_I1_TERMINATOR:
10894 return true;
10895 default:
10896 return false;
10897 }
10898}
10899
10901 switch (Opcode) {
10902 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
10903 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
10904 case AMDGPU::SI_KILL_I1_PSEUDO:
10905 return get(AMDGPU::SI_KILL_I1_TERMINATOR);
10906 default:
10907 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
10908 }
10909}
10910
10912 return Imm <= getMaxMUBUFImmOffset(ST);
10913}
10914
10916 // GFX12 field is non-negative 24-bit signed byte offset.
10917 const unsigned OffsetBits =
10918 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
10919 return (1 << OffsetBits) - 1;
10920}
10921
10923 if (!ST.isWave32())
10924 return;
10925
10926 if (MI.isInlineAsm())
10927 return;
10928
10929 if (MI.getNumOperands() < MI.getDesc().getNumOperands())
10930 return;
10931
10932 for (auto &Op : MI.implicit_operands()) {
10933 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
10934 Op.setReg(AMDGPU::VCC_LO);
10935 }
10936}
10937
10939 if (!isSMRD(MI))
10940 return false;
10941
10942 // Check that it is using a buffer resource.
10943 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);
10944 if (Idx == -1) // e.g. s_memtime
10945 return false;
10946
10947 const int16_t RCID = getOpRegClassID(MI.getDesc().operands()[Idx]);
10948 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass);
10949}
10950
10951// Given Imm, split it into the values to put into the SOffset and ImmOffset
10952// fields in an MUBUF instruction. Return false if it is not possible (due to a
10953// hardware bug needing a workaround).
10954//
10955// The required alignment ensures that individual address components remain
10956// aligned if they are aligned to begin with. It also ensures that additional
10957// offsets within the given alignment can be added to the resulting ImmOffset.
10959 uint32_t &ImmOffset, Align Alignment) const {
10960 const uint64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
10961 const uint32_t MaxImm = alignDown(MaxOffset, Alignment.value());
10962 uint32_t Overflow = 0;
10963
10964 if (Imm > MaxImm) {
10965 if (Imm <= MaxImm + 64) {
10966 // Use an SOffset inline constant for 4..64
10967 Overflow = Imm - MaxImm;
10968 Imm = MaxImm;
10969 } else {
10970 // Try to keep the same value in SOffset for adjacent loads, so that
10971 // the corresponding register contents can be re-used.
10972 //
10973 // Load values with all low-bits (except for alignment bits) set into
10974 // SOffset, so that a larger range of values can be covered using
10975 // s_movk_i32.
10976 //
10977 // Atomic operations fail to work correctly when individual address
10978 // components are unaligned, even if their sum is aligned.
10979 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
10980 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
10981 Imm = Low;
10982 Overflow = High - Alignment.value();
10983 }
10984 }
10985
10986 if (Overflow > 0) {
10987 // There is a hardware bug in SI and CI which prevents address clamping in
10988 // MUBUF instructions from working correctly with SOffsets. The immediate
10989 // offset is unaffected.
10990 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
10991 return false;
10992
10993 // It is not possible to set immediate in SOffset field on some targets.
10994 if (ST.hasRestrictedSOffset())
10995 return false;
10996 }
10997
10998 ImmOffset = Imm;
10999 SOffset = Overflow;
11000 return true;
11001}
11002
11003// Depending on the used address space and instructions, some immediate offsets
11004// are allowed and some are not.
11005// Pre-GFX12, flat instruction offsets can only be non-negative, global and
11006// scratch instruction offsets can also be negative. On GFX12, offsets can be
11007// negative for all variants.
11008//
11009// There are several bugs related to these offsets:
11010// On gfx10.1, flat instructions that go into the global address space cannot
11011// use an offset.
11012//
11013// For scratch instructions, the address can be either an SGPR or a VGPR.
11014// The following offsets can be used, depending on the architecture (x means
11015// cannot be used):
11016// +----------------------------+------+------+
11017// | Address-Mode | SGPR | VGPR |
11018// +----------------------------+------+------+
11019// | gfx9 | | |
11020// | negative, 4-aligned offset | x | ok |
11021// | negative, unaligned offset | x | ok |
11022// +----------------------------+------+------+
11023// | gfx10 | | |
11024// | negative, 4-aligned offset | ok | ok |
11025// | negative, unaligned offset | ok | x |
11026// +----------------------------+------+------+
11027// | gfx10.3 | | |
11028// | negative, 4-aligned offset | ok | ok |
11029// | negative, unaligned offset | ok | ok |
11030// +----------------------------+------+------+
11031//
11032// This function ignores the addressing mode, so if an offset cannot be used in
11033// one addressing mode, it is considered illegal.
11034bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
11035 AMDGPU::FlatAddrSpace FlatVariant) const {
11036 // TODO: Should 0 be special cased?
11037 if (!ST.hasFlatInstOffsets())
11038 return false;
11039
11041 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == FlatAddrSpace::FLAT &&
11042 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
11044 return false;
11045
11046 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
11047 FlatVariant == FlatAddrSpace::FlatScratch && Offset < 0 &&
11048 (Offset % 4) != 0) {
11049 return false;
11050 }
11051
11052 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
11053 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
11054 return isIntN(N, Offset) && (AllowNegative || Offset >= 0);
11055}
11056
11057// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
11058std::pair<int64_t, int64_t>
11059SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
11060 AMDGPU::FlatAddrSpace FlatVariant) const {
11061 int64_t RemainderOffset = COffsetVal;
11062 int64_t ImmField = 0;
11063
11064 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
11065 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
11066
11067 if (AllowNegative) {
11068 // Use signed division by a power of two to truncate towards 0.
11069 int64_t D = 1LL << NumBits;
11070 RemainderOffset = (COffsetVal / D) * D;
11071 ImmField = COffsetVal - RemainderOffset;
11072
11073 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
11074 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch && ImmField < 0 &&
11075 (ImmField % 4) != 0) {
11076 // Make ImmField a multiple of 4
11077 RemainderOffset += ImmField % 4;
11078 ImmField -= ImmField % 4;
11079 }
11080 } else if (COffsetVal >= 0) {
11081 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits);
11082 RemainderOffset = COffsetVal - ImmField;
11083 }
11084
11085 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
11086 assert(RemainderOffset + ImmField == COffsetVal);
11087 return {ImmField, RemainderOffset};
11088}
11089
11091 AMDGPU::FlatAddrSpace FlatVariant) const {
11092 if (ST.hasNegativeScratchOffsetBug() &&
11094 return false;
11095
11096 return FlatVariant != AMDGPU::FlatAddrSpace::FLAT || AMDGPU::isGFX12Plus(ST);
11097}
11098
11099static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
11100 switch (ST.getGeneration()) {
11101 default:
11102 break;
11105 return SIEncodingFamily::SI;
11107 // The GFX80 encoding family only contains buffer instructions with unpacked
11108 // D16 data; pseudoToMCOpcode falls back on VI for everything else.
11109 // TODO: remove this when we discard GFX80 encoding.
11110 return ST.hasUnpackedD16VMem() ? SIEncodingFamily::GFX80
11113 return SIEncodingFamily::VI;
11117 return ST.hasGFX11_7Insts() ? SIEncodingFamily::GFX1170
11120 return ST.hasGFX1250Insts() ? SIEncodingFamily::GFX1250
11124 }
11125 llvm_unreachable("Unknown subtarget generation!");
11126}
11127
11128bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
11129 switch(MCOp) {
11130 // These opcodes use indirect register addressing so
11131 // they need special handling by codegen (currently missing).
11132 // Therefore it is too risky to allow these opcodes
11133 // to be selected by dpp combiner or sdwa peepholer.
11134 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
11135 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
11136 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
11137 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
11138 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
11139 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
11140 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
11141 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
11142 return true;
11143 default:
11144 return false;
11145 }
11146}
11147
11148#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \
11149 case OPCODE##_dpp: \
11150 case OPCODE##_e32: \
11151 case OPCODE##_e64:
11152
11153static bool isRenamedInGFX9(int Opcode) {
11154 switch (Opcode) {
11155 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
11156 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
11157 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
11158 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
11159 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
11160 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
11161 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
11162 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
11163 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
11164 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
11165 case AMDGPU::V_FMA_F16_gfx9_e64:
11166 case AMDGPU::V_INTERP_P2_F16:
11167 case AMDGPU::V_MAD_F16_e64:
11168 case AMDGPU::V_MAD_U16_e64:
11169 case AMDGPU::V_MAD_I16_e64:
11170 case AMDGPU::V_MAD_U16_gfx9_e64:
11171 case AMDGPU::V_MAD_I16_gfx9_e64:
11172 return true;
11173 default:
11174 return false;
11175 }
11176}
11177
11178int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
11179 assert(Opcode == (int)SIInstrInfo::getNonSoftWaitcntOpcode(Opcode) &&
11180 "SIInsertWaitcnts should have promoted soft waitcnt instructions!");
11181
11182 unsigned Gen = subtargetEncodingFamily(ST);
11183
11184 if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))
11186
11187 if (SIInstrFlags::isSDWA(get(Opcode))) {
11188 switch (ST.getGeneration()) {
11189 default:
11191 break;
11194 break;
11197 break;
11198 }
11199 }
11200
11201 if (isMAI(Opcode)) {
11202 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
11203 if (MFMAOp != -1)
11204 Opcode = MFMAOp;
11205 }
11206
11207 int32_t MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
11208
11209 // Only buffer instructions with unpacked D16 data have a GFX80 encoding.
11210 // Anything else on such a subtarget uses the plain VI encoding.
11211 // TODO: remove this when we discard GFX80 encoding.
11212 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && Gen == SIEncodingFamily::GFX80)
11214
11215 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX11_7Insts())
11217
11218 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX1250Insts())
11220
11221 // -1 means that Opcode is already a native instruction.
11222 if (MCOp == -1)
11223 return Opcode;
11224
11225 if (ST.hasGFX90AInsts()) {
11226 uint32_t NMCOp = AMDGPU::INSTRUCTION_LIST_END;
11227 if (ST.hasGFX940Insts())
11229 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11231 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11233 if (NMCOp != AMDGPU::INSTRUCTION_LIST_END)
11234 MCOp = NMCOp;
11235 }
11236
11237 // INSTRUCTION_LIST_END means that Opcode is a pseudo instruction that has no
11238 // encoding in the given subtarget generation.
11239 if (MCOp == AMDGPU::INSTRUCTION_LIST_END)
11240 return -1;
11241
11242 if (isAsmOnlyOpcode(MCOp))
11243 return -1;
11244
11245 return MCOp;
11246}
11247
11248static
11250 assert(RegOpnd.isReg());
11251 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
11252 getRegSubRegPair(RegOpnd);
11253}
11254
11257 assert(MI.isRegSequence());
11258 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
11259 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) {
11260 auto &RegOp = MI.getOperand(1 + 2 * I);
11261 return getRegOrUndef(RegOp);
11262 }
11264}
11265
11266// Try to find the definition of reg:subreg in subreg-manipulation pseudos
11267// Following a subreg of reg:subreg isn't supported
11270 if (!RSR.SubReg)
11271 return false;
11272 switch (MI.getOpcode()) {
11273 default: break;
11274 case AMDGPU::REG_SEQUENCE:
11275 RSR = getRegSequenceSubReg(MI, RSR.SubReg);
11276 return true;
11277 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
11278 case AMDGPU::INSERT_SUBREG:
11279 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm())
11280 // inserted the subreg we're looking for
11281 RSR = getRegOrUndef(MI.getOperand(2));
11282 else { // the subreg in the rest of the reg
11283 auto R1 = getRegOrUndef(MI.getOperand(1));
11284 if (R1.SubReg) // subreg of subreg isn't supported
11285 return false;
11286 RSR.Reg = R1.Reg;
11287 }
11288 return true;
11289 }
11290 return false;
11291}
11292
11294 const MachineRegisterInfo &MRI) {
11295 assert(MRI.isSSA());
11296 if (!P.Reg.isVirtual())
11297 return nullptr;
11298
11299 auto RSR = P;
11300 auto *DefInst = MRI.getVRegDef(RSR.Reg);
11301 while (auto *MI = DefInst) {
11302 DefInst = nullptr;
11303 switch (MI->getOpcode()) {
11304 case AMDGPU::COPY:
11305 case AMDGPU::V_MOV_B32_e32: {
11306 auto &Op1 = MI->getOperand(1);
11307 if (Op1.isReg() && Op1.getReg().isVirtual()) {
11308 if (Op1.isUndef())
11309 return nullptr;
11310 RSR = getRegSubRegPair(Op1);
11311 DefInst = MRI.getVRegDef(RSR.Reg);
11312 }
11313 break;
11314 }
11315 default:
11316 if (followSubRegDef(*MI, RSR)) {
11317 if (!RSR.Reg)
11318 return nullptr;
11319 DefInst = MRI.getVRegDef(RSR.Reg);
11320 }
11321 }
11322 if (!DefInst)
11323 return MI;
11324 }
11325 return nullptr;
11326}
11327
11329 Register VReg,
11330 const MachineInstr &DefMI,
11331 const MachineInstr &UseMI) {
11332 assert(MRI.isSSA() && "Must be run on SSA");
11333
11334 auto *TRI = MRI.getTargetRegisterInfo();
11335 auto *DefBB = DefMI.getParent();
11336
11337 // Don't bother searching between blocks, although it is possible this block
11338 // doesn't modify exec.
11339 if (UseMI.getParent() != DefBB)
11340 return true;
11341
11342 const int MaxInstScan = 20;
11343 int NumInst = 0;
11344
11345 // Stop scan at the use.
11346 auto E = UseMI.getIterator();
11347 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) {
11348 if (I->isDebugInstr())
11349 continue;
11350
11351 if (++NumInst > MaxInstScan)
11352 return true;
11353
11354 if (I->modifiesRegister(AMDGPU::EXEC, TRI))
11355 return true;
11356 }
11357
11358 return false;
11359}
11360
11362 Register VReg,
11363 const MachineInstr &DefMI) {
11364 assert(MRI.isSSA() && "Must be run on SSA");
11365
11366 auto *TRI = MRI.getTargetRegisterInfo();
11367 auto *DefBB = DefMI.getParent();
11368
11369 const int MaxUseScan = 10;
11370 int NumUse = 0;
11371
11372 for (auto &Use : MRI.use_nodbg_operands(VReg)) {
11373 auto &UseInst = *Use.getParent();
11374 // Don't bother searching between blocks, although it is possible this block
11375 // doesn't modify exec.
11376 if (UseInst.getParent() != DefBB || UseInst.isPHI())
11377 return true;
11378
11379 if (++NumUse > MaxUseScan)
11380 return true;
11381 }
11382
11383 if (NumUse == 0)
11384 return false;
11385
11386 const int MaxInstScan = 20;
11387 int NumInst = 0;
11388
11389 // Stop scan when we have seen all the uses.
11390 for (auto I = std::next(DefMI.getIterator()); ; ++I) {
11391 assert(I != DefBB->end());
11392
11393 if (I->isDebugInstr())
11394 continue;
11395
11396 if (++NumInst > MaxInstScan)
11397 return true;
11398
11399 for (const MachineOperand &Op : I->operands()) {
11400 // We don't check reg masks here as they're used only on calls:
11401 // 1. EXEC is only considered const within one BB
11402 // 2. Call should be a terminator instruction if present in a BB
11403
11404 if (!Op.isReg())
11405 continue;
11406
11407 Register Reg = Op.getReg();
11408 if (Op.isUse()) {
11409 if (Reg == VReg && --NumUse == 0)
11410 return false;
11411 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC))
11412 return true;
11413 }
11414 }
11415}
11416
11419 const DebugLoc &DL, Register Src, Register Dst) const {
11420 auto Cur = MBB.begin();
11421 if (Cur != MBB.end())
11422 do {
11423 if (!Cur->isPHI() && Cur->readsRegister(Dst, /*TRI=*/nullptr))
11424 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src);
11425 ++Cur;
11426 } while (Cur != MBB.end() && Cur != LastPHIIt);
11427
11428 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src,
11429 Dst);
11430}
11431
11434 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
11435 if (InsPt != MBB.end() &&
11436 (InsPt->getOpcode() == AMDGPU::SI_IF ||
11437 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
11438 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
11439 InsPt->definesRegister(Src, /*TRI=*/nullptr)) {
11440 InsPt++;
11441 return BuildMI(MBB, InsPt, DL,
11442 get(AMDGPU::LaneMaskConstants::get(ST).MovTermOpc), Dst)
11443 .addReg(Src, {}, SrcSubReg)
11444 .addReg(AMDGPU::EXEC, RegState::Implicit);
11445 }
11446 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
11447 Dst);
11448}
11449
11450bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
11451
11453 const MachineInstr &SecondMI) const {
11454 for (const auto &Use : SecondMI.all_uses()) {
11455 if (Use.isReg() && FirstMI.modifiesRegister(Use.getReg(), &RI))
11456 return true;
11457 }
11458 return false;
11459}
11460
11461/// If OpX is multicycle, anti-dependencies are not allowed.
11462/// isDPMACCInstruction was not designed for VOPD, but it is fit for the
11463/// purpose.
11465 const MachineInstr &OpX) const {
11467}
11468
11471 ArrayRef<unsigned> Ops, int FrameIndex,
11472 MachineInstr *&CopyMI, LiveIntervals *LIS,
11473 VirtRegMap *VRM) const {
11474 // This is a bit of a hack (copied from AArch64). Consider this instruction:
11475 //
11476 // %0:sreg_32 = COPY $m0
11477 //
11478 // We explicitly chose SReg_32 for the virtual register so such a copy might
11479 // be eliminated by RegisterCoalescer. However, that may not be possible, and
11480 // %0 may even spill. We can't spill $m0 normally (it would require copying to
11481 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
11482 // TargetInstrInfo::foldMemoryOperand() is going to try.
11483 // A similar issue also exists with spilling and reloading $exec registers.
11484 //
11485 // To prevent that, constrain the %0 register class here.
11486 if (isFullCopyInstr(MI)) {
11487 Register DstReg = MI.getOperand(0).getReg();
11488 Register SrcReg = MI.getOperand(1).getReg();
11489 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
11490 (DstReg.isVirtual() != SrcReg.isVirtual())) {
11492 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
11493 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg);
11494 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) {
11495 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
11496 return nullptr;
11497 }
11498 if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) {
11499 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass);
11500 return nullptr;
11501 }
11502 }
11503 }
11504
11505 return nullptr;
11506}
11507
11509 const MachineInstr &MI,
11510 unsigned *PredCost) const {
11511 if (MI.isBundle()) {
11513 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
11514 unsigned Lat = 0, Count = 0;
11515 for (++I; I != E && I->isBundledWithPred(); ++I) {
11516 ++Count;
11517 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I));
11518 }
11519 return Lat + Count - 1;
11520 }
11521
11522 return SchedModel.computeInstrLatency(&MI);
11523}
11524
11526 if (!ST.hasGFX1250VALUBlockingCycles())
11527 return 0;
11529}
11530
11531unsigned
11533 if (const auto *Entry = AMDGPU::getGFX1250BlockingCyclesInfo(MI.getOpcode()))
11534 return Entry->GFX1250BlockingCycles;
11535 return 0;
11536}
11537
11538const MachineOperand &
11540 if (const MachineOperand *CallAddrOp =
11541 getNamedOperand(MI, AMDGPU::OpName::src0))
11542 return *CallAddrOp;
11544}
11545
11548 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11549 unsigned Opcode = MI.getOpcode();
11550
11551 auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {
11552 Register Dst = MI.getOperand(0).getReg();
11553 Register Src = MI.getOperand(1).getReg();
11554 LLT DstTy = MRI.getType(Dst);
11555 LLT SrcTy = MRI.getType(Src);
11556 unsigned DstAS = DstTy.getAddressSpace();
11557 unsigned SrcAS = SrcTy.getAddressSpace();
11558 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
11559 DstAS == AMDGPUAS::FLAT_ADDRESS &&
11560 ST.hasGloballyAddressableScratch()
11563 };
11564
11565 // If the target supports globally addressable scratch, the mapping from
11566 // scratch memory to the flat aperture changes therefore an address space cast
11567 // is no longer uniform.
11568 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
11569 return HandleAddrSpaceCast(MI);
11570
11571 if (auto *GI = dyn_cast<GIntrinsic>(&MI)) {
11572 auto IID = GI->getIntrinsicID();
11577
11578 switch (IID) {
11579 case Intrinsic::amdgcn_if:
11580 case Intrinsic::amdgcn_else:
11581 // FIXME: Uniform if second result
11582 break;
11583 }
11584
11586 }
11587
11588 // Each lane reads its own registers, however uniform the index.
11589 if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
11591
11592 // Loads from the private and flat address spaces are divergent, because
11593 // threads can execute the load instruction with the same inputs and get
11594 // different results. So are VGPR address space loads, before they become
11595 // G_AMDGPU_REG_LOAD.
11596 //
11597 // All other loads are not divergent, because if threads issue loads with the
11598 // same arguments, they will always get the same result.
11599 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||
11600 Opcode == AMDGPU::G_SEXTLOAD) {
11601 if (MI.memoperands_empty())
11602 return ValueUniformity::NeverUniform; // conservative assumption
11603
11604 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) {
11605 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11606 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS ||
11607 mmo->getAddrSpace() == AMDGPUAS::VGPR;
11608 })) {
11609 // At least one MMO in a non-global address space.
11611 }
11613 }
11614
11615 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opcode) ||
11616 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
11617 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
11618 AMDGPU::isGenericAtomic(Opcode)) {
11620 }
11621
11622 // Result is computed from uniform SP and uniform wave-wide max size.
11623 if (Opcode == TargetOpcode::G_DYN_STACKALLOC)
11625
11626 if (Opcode == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
11628
11630}
11631
11633 if (!Formatter)
11634 Formatter = std::make_unique<AMDGPUMIRFormatter>(ST);
11635 return Formatter.get();
11636}
11637
11639
11640 if (isNeverUniform(MI))
11642
11643 unsigned opcode = MI.getOpcode();
11644 if (opcode == AMDGPU::V_READLANE_B32 ||
11645 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
11646 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
11648
11649 // If any of defs is divergent, report as NeverUniform. isUniformReg will
11650 // calculate in more detail for each def from its reg class, if available.
11651 if (MI.isInlineAsm()) {
11652 for (const MachineOperand &MO : MI.operands()) {
11653 if (!MO.isReg() || !MO.isDef())
11654 continue;
11655 const TargetRegisterClass *RC =
11656 MI.getRegClassConstraint(MO.getOperandNo(), this, &RI);
11657 if (!RC || !RI.isSGPRClass(RC))
11659 }
11660 }
11661
11662 if (isCopyInstr(MI)) {
11663 const MachineOperand &srcOp = MI.getOperand(1);
11664 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
11665 const TargetRegisterClass *regClass =
11666 RI.getPhysRegBaseClass(srcOp.getReg());
11667 return RI.isSGPRClass(regClass) ? ValueUniformity::AlwaysUniform
11669 }
11671 }
11672
11673 // GMIR handling
11674 if (MI.isPreISelOpcode())
11676
11677 // Atomics are divergent because they are executed sequentially: when an
11678 // atomic operation refers to the same address in each thread, then each
11679 // thread after the first sees the value written by the previous thread as
11680 // original value.
11681
11682 if (isAtomic(MI))
11684
11685 // Loads from the private and flat address spaces are divergent, because
11686 // threads can execute the load instruction with the same inputs and get
11687 // different results.
11688 if (isFLAT(MI) && MI.mayLoad()) {
11689 if (MI.memoperands_empty())
11690 return ValueUniformity::NeverUniform; // conservative assumption
11691
11692 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) {
11693 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11694 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11695 })) {
11696 // At least one MMO in a non-global address space.
11698 }
11699
11701 }
11702
11703 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11704 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
11705
11706 // FIXME: It's conceptually broken to report this for an instruction, and not
11707 // a specific def operand. For inline asm in particular, there could be mixed
11708 // uniform and divergent results.
11709 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
11710 const MachineOperand &SrcOp = MI.getOperand(I);
11711 if (!SrcOp.isReg())
11712 continue;
11713
11714 Register Reg = SrcOp.getReg();
11715 if (!Reg || !SrcOp.readsReg())
11716 continue;
11717
11718 // If RegBank is null, this is unassigned or an unallocatable special
11719 // register, which are all scalars.
11720 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, RI);
11721 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
11723 }
11724
11725 // TODO: Uniformity check condtions above can be rearranged for more
11726 // redability
11727
11728 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
11729 // currently turned into no-op COPYs by SelectionDAG ISel and are
11730 // therefore no longer recognizable.
11731
11733}
11734
11736 switch (MF.getFunction().getCallingConv()) {
11738 return 1;
11740 return 2;
11742 return 3;
11746 const Function &F = MF.getFunction();
11747 F.getContext().diagnose(DiagnosticInfoUnsupported(
11748 F, "ds_ordered_count unsupported for this calling conv"));
11749 [[fallthrough]];
11750 }
11753 case CallingConv::C:
11754 case CallingConv::Fast:
11755 default:
11756 // Assume other calling conventions are various compute callable functions
11757 return 0;
11758 }
11759}
11760
11762 Register &SrcReg2, int64_t &CmpMask,
11763 int64_t &CmpValue) const {
11764 if (!MI.getOperand(0).isReg() || MI.getOperand(0).getSubReg())
11765 return false;
11766
11767 switch (MI.getOpcode()) {
11768 default:
11769 break;
11770 case AMDGPU::S_CMP_EQ_U32:
11771 case AMDGPU::S_CMP_EQ_I32:
11772 case AMDGPU::S_CMP_LG_U32:
11773 case AMDGPU::S_CMP_LG_I32:
11774 case AMDGPU::S_CMP_LT_U32:
11775 case AMDGPU::S_CMP_LT_I32:
11776 case AMDGPU::S_CMP_GT_U32:
11777 case AMDGPU::S_CMP_GT_I32:
11778 case AMDGPU::S_CMP_LE_U32:
11779 case AMDGPU::S_CMP_LE_I32:
11780 case AMDGPU::S_CMP_GE_U32:
11781 case AMDGPU::S_CMP_GE_I32:
11782 case AMDGPU::S_CMP_EQ_U64:
11783 case AMDGPU::S_CMP_LG_U64:
11784 SrcReg = MI.getOperand(0).getReg();
11785 if (MI.getOperand(1).isReg()) {
11786 if (MI.getOperand(1).getSubReg())
11787 return false;
11788 SrcReg2 = MI.getOperand(1).getReg();
11789 CmpValue = 0;
11790 } else if (MI.getOperand(1).isImm()) {
11791 SrcReg2 = Register();
11792 CmpValue = MI.getOperand(1).getImm();
11793 } else {
11794 return false;
11795 }
11796 CmpMask = ~0;
11797 return true;
11798 case AMDGPU::S_CMPK_EQ_U32:
11799 case AMDGPU::S_CMPK_EQ_I32:
11800 case AMDGPU::S_CMPK_LG_U32:
11801 case AMDGPU::S_CMPK_LG_I32:
11802 case AMDGPU::S_CMPK_LT_U32:
11803 case AMDGPU::S_CMPK_LT_I32:
11804 case AMDGPU::S_CMPK_GT_U32:
11805 case AMDGPU::S_CMPK_GT_I32:
11806 case AMDGPU::S_CMPK_LE_U32:
11807 case AMDGPU::S_CMPK_LE_I32:
11808 case AMDGPU::S_CMPK_GE_U32:
11809 case AMDGPU::S_CMPK_GE_I32:
11810 SrcReg = MI.getOperand(0).getReg();
11811 SrcReg2 = Register();
11812 CmpValue = MI.getOperand(1).getImm();
11813 CmpMask = ~0;
11814 return true;
11815 }
11816
11817 return false;
11818}
11819
11821 for (MachineBasicBlock *S : MBB->successors()) {
11822 if (S->isLiveIn(AMDGPU::SCC))
11823 return false;
11824 }
11825 return true;
11826}
11827
11828// Invert all uses of SCC following SCCDef because SCCDef may be deleted and
11829// (incoming SCC) = !(SCC defined by SCCDef).
11830// Return true if all uses can be re-written, false otherwise.
11831bool SIInstrInfo::invertSCCUse(MachineInstr *SCCDef) const {
11832 MachineBasicBlock *MBB = SCCDef->getParent();
11833 SmallVector<MachineInstr *> InvertInstr;
11834 bool SCCIsDead = false;
11835
11836 // Scan instructions for SCC uses that need to be inverted until SCC is dead.
11837 constexpr unsigned ScanLimit = 12;
11838 unsigned Count = 0;
11839 for (MachineInstr &MI : instructionsWithoutDebug(
11840 std::next(MachineBasicBlock::iterator(SCCDef)), MBB->end())) {
11841 if (++Count > ScanLimit)
11842 return false;
11843 if (MI.readsRegister(AMDGPU::SCC, &RI)) {
11844 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
11845 MI.getOpcode() == AMDGPU::S_CSELECT_B64 ||
11846 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11847 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC1)
11848 InvertInstr.push_back(&MI);
11849 else
11850 return false;
11851 }
11852 if (MI.definesRegister(AMDGPU::SCC, &RI)) {
11853 SCCIsDead = true;
11854 break;
11855 }
11856 }
11857 if (!SCCIsDead && isSCCDeadOnExit(MBB))
11858 SCCIsDead = true;
11859
11860 // SCC may have more uses. Can't invert all of them.
11861 if (!SCCIsDead)
11862 return false;
11863
11864 // Invert uses
11865 for (MachineInstr *MI : InvertInstr) {
11866 if (MI->getOpcode() == AMDGPU::S_CSELECT_B32 ||
11867 MI->getOpcode() == AMDGPU::S_CSELECT_B64) {
11868 swapOperands(*MI);
11869 } else if (MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11870 MI->getOpcode() == AMDGPU::S_CBRANCH_SCC1) {
11871 MI->setDesc(get(MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0
11872 ? AMDGPU::S_CBRANCH_SCC1
11873 : AMDGPU::S_CBRANCH_SCC0));
11874 } else {
11875 llvm_unreachable("SCC used but no inversion handling");
11876 }
11877 }
11878 return true;
11879}
11880
11881// SCC is already valid after SCCValid.
11882// SCCRedefine will redefine SCC to the same value already available after
11883// SCCValid. If there are no intervening SCC conflicts delete SCCRedefine and
11884// update kill/dead flags if necessary.
11885bool SIInstrInfo::optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
11886 bool NeedInversion) const {
11887 MachineInstr *KillsSCC = nullptr;
11888 if (SCCValid->getParent() != SCCRedefine->getParent())
11889 return false;
11890 for (MachineInstr &MI : make_range(std::next(SCCValid->getIterator()),
11891 SCCRedefine->getIterator())) {
11892 if (MI.modifiesRegister(AMDGPU::SCC, &RI))
11893 return false;
11894 if (MI.killsRegister(AMDGPU::SCC, &RI))
11895 KillsSCC = &MI;
11896 }
11897 if (NeedInversion && !invertSCCUse(SCCRedefine))
11898 return false;
11899 if (MachineOperand *SccDef =
11900 SCCValid->findRegisterDefOperand(AMDGPU::SCC, /*TRI=*/nullptr))
11901 SccDef->setIsDead(false);
11902 if (KillsSCC)
11903 KillsSCC->clearRegisterKills(AMDGPU::SCC, /*TRI=*/nullptr);
11904 SCCRedefine->eraseFromParent();
11905 return true;
11906}
11907
11908/// If \p Sel is an S_CSELECT* of two different constants A and B, return them,
11909/// truncated to the width of the select.
11910static std::optional<std::pair<int64_t, int64_t>>
11912 const MachineInstr &Sel) {
11913 unsigned Opc = Sel.getOpcode();
11914 if (Opc != AMDGPU::S_CSELECT_B32 && Opc != AMDGPU::S_CSELECT_B64)
11915 return {};
11916 std::optional<int64_t> A =
11917 TII.getImmOrMaterializedImm(MRI, Sel.getOperand(1));
11918 if (!A)
11919 return {};
11920 std::optional<int64_t> B =
11921 TII.getImmOrMaterializedImm(MRI, Sel.getOperand(2));
11922 if (!B)
11923 return {};
11924 if (Opc == AMDGPU::S_CSELECT_B32) {
11925 A = Lo_32(*A);
11926 B = Lo_32(*B);
11927 }
11928 if (*A == *B)
11929 return {};
11930 return std::pair(*A, *B);
11931}
11932
11933static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion,
11934 unsigned &NewDefOpc) {
11935 // S_ADD_U32 X, 1 sets SCC on carryout which can only happen if result==0.
11936 // S_ADD_I32 X, 1 can be converted to S_ADD_U32 X, 1 if SCC is dead.
11937 if (Def.getOpcode() != AMDGPU::S_ADD_I32 &&
11938 Def.getOpcode() != AMDGPU::S_ADD_U32)
11939 return false;
11940 const MachineOperand &AddSrc1 = Def.getOperand(1);
11941 const MachineOperand &AddSrc2 = Def.getOperand(2);
11942 const MachineRegisterInfo &MRI = Def.getMF()->getRegInfo();
11943 const SIInstrInfo *TII = static_cast<const SIInstrInfo *>(
11944 Def.getMF()->getSubtarget().getInstrInfo());
11945
11946 auto Imm1 = TII->getImmOrMaterializedImm(MRI, AddSrc1);
11947 auto Imm2 = TII->getImmOrMaterializedImm(MRI, AddSrc2);
11948 if ((!Imm1 || *Imm1 != 1) && (!Imm2 || *Imm2 != 1))
11949 return false;
11950
11951 if (Def.getOpcode() == AMDGPU::S_ADD_I32) {
11952 const MachineOperand *SccDef =
11953 Def.findRegisterDefOperand(AMDGPU::SCC, /*TRI=*/nullptr);
11954 if (!SccDef->isDead())
11955 return false;
11956 NewDefOpc = AMDGPU::S_ADD_U32;
11957 }
11958 NeedInversion = !NeedInversion;
11959 return true;
11960}
11961
11963 Register SrcReg2, int64_t CmpMask,
11964 int64_t CmpValue,
11965 const MachineRegisterInfo *MRI) const {
11966 if (!SrcReg || SrcReg.isPhysical())
11967 return false;
11968
11969 if (SrcReg2) {
11970 auto ImmOpt = getImmOrMaterializedImm(*MRI, SrcReg2);
11971 if (!ImmOpt)
11972 return false;
11973 CmpValue = *ImmOpt;
11974 }
11975
11976 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,
11977 this](bool NeedInversion) -> bool {
11978 MachineInstr *Def = MRI->getVRegDef(SrcReg);
11979 if (!Def)
11980 return false;
11981
11982 unsigned NewDefOpc = Def->getOpcode();
11983 if (auto Consts = getSelectConstants(*this, *MRI, *Def)) {
11984 // sX = S_CSELECT* A, B with A != B, so sX == A exactly when SCC was set.
11985 // Comparing sX with A or B recomputes SCC or its inverse:
11986 //
11987 // s_cmp_eq_* sX, A => SCC s_cmp_lg_* sX, A => !SCC
11988 // s_cmp_eq_* sX, B => !SCC s_cmp_lg_* sX, B => SCC
11989 auto [A, B] = *Consts;
11990 int64_t C = Def->getOpcode() == AMDGPU::S_CSELECT_B32 ? Lo_32(CmpValue)
11991 : CmpValue;
11992 if (C == A)
11993 NeedInversion = !NeedInversion;
11994 else if (C != B)
11995 return false;
11996 } else {
11997 // For S_OP that set SCC = DST!=0, do the transformation
11998 //
11999 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
12000 //
12001 // For (S_OP ...) that set SCC = DST==0, invert NeedInversion and
12002 // do the transformation:
12003 //
12004 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
12005 if (CmpValue != 0 ||
12006 (!setsSCCIfResultIsNonZero(*Def) &&
12007 !setsSCCIfResultIsZero(*Def, NeedInversion, NewDefOpc)))
12008 return false;
12009 }
12010
12011 if (!optimizeSCC(Def, &CmpInstr, NeedInversion))
12012 return false;
12013
12014 if (NewDefOpc != Def->getOpcode())
12015 Def->setDesc(get(NewDefOpc));
12016
12017 // If s_or_b32 result, sY, is unused (i.e. it is effectively a 64-bit
12018 // s_cmp_lg of a register pair) and the inputs are the hi and lo-halves of a
12019 // 64-bit select then delete s_or_b32 in the sequence:
12020 // sX = s_cselect_b64 A, B (A != B, one of them 0)
12021 // sLo = copy sX.sub0
12022 // sHi = copy sX.sub1
12023 // sY = s_or_b32 sLo, sHi
12024 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&
12025 MRI->use_nodbg_empty(Def->getOperand(0).getReg())) {
12026 const MachineOperand &OrOpnd1 = Def->getOperand(1);
12027 const MachineOperand &OrOpnd2 = Def->getOperand(2);
12028 if (OrOpnd1.isReg() && OrOpnd2.isReg()) {
12029 MachineInstr *Def1 = MRI->getVRegDef(OrOpnd1.getReg());
12030 MachineInstr *Def2 = MRI->getVRegDef(OrOpnd2.getReg());
12031 if (Def1 && Def1->getOpcode() == AMDGPU::COPY && Def2 &&
12032 Def2->getOpcode() == AMDGPU::COPY && Def1->getOperand(1).isReg() &&
12033 Def2->getOperand(1).isReg() &&
12034 Def1->getOperand(1).getSubReg() == AMDGPU::sub0 &&
12035 Def2->getOperand(1).getSubReg() == AMDGPU::sub1 &&
12036 Def1->getOperand(1).getReg() == Def2->getOperand(1).getReg()) {
12037 if (MachineInstr *Select =
12038 MRI->getVRegDef(Def1->getOperand(1).getReg())) {
12039 if (auto Consts = getSelectConstants(*this, *MRI, *Select)) {
12040 auto [A, B] = *Consts;
12041 if (A == 0 || B == 0)
12042 optimizeSCC(Select, Def, /*NeedInversion=*/A == 0);
12043 }
12044 }
12045 }
12046 }
12047 }
12048 return true;
12049 };
12050
12051 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
12052 this](int64_t ExpectedValue, unsigned SrcSize,
12053 bool IsReversible, bool IsSigned) -> bool {
12054 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12055 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12056 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12057 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12058 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
12059 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12060 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12061 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12062 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12063 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
12064 //
12065 // Signed ge/gt are not used for the sign bit.
12066 //
12067 // If result of the AND is unused except in the compare:
12068 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
12069 //
12070 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
12071 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
12072 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
12073 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
12074 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
12075 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
12076
12077 MachineInstr *Def = MRI->getVRegDef(SrcReg);
12078 if (!Def)
12079 return false;
12080
12081 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
12082 Def->getOpcode() != AMDGPU::S_AND_B64)
12083 return false;
12084
12085 int64_t Mask;
12086 const auto isMask = [&Mask, SrcSize, MRI,
12087 this](const MachineOperand *MO) -> bool {
12088 auto ImmOpt = this->getImmOrMaterializedImm(*MRI, *MO);
12089 if (!ImmOpt)
12090 return false;
12091 Mask = *ImmOpt;
12092 Mask &= maxUIntN(SrcSize);
12093 return isPowerOf2_64(Mask);
12094 };
12095
12096 MachineOperand *SrcOp = &Def->getOperand(1);
12097 if (isMask(SrcOp))
12098 SrcOp = &Def->getOperand(2);
12099 else if (isMask(&Def->getOperand(2)))
12100 SrcOp = &Def->getOperand(1);
12101 else
12102 return false;
12103
12104 // A valid Mask is required to have a single bit set, hence a non-zero and
12105 // power-of-two value. This verifies that we will not do 64-bit shift below.
12106 assert(llvm::has_single_bit<uint64_t>(Mask) && "Invalid mask.");
12107 unsigned BitNo = llvm::countr_zero((uint64_t)Mask);
12108 if (IsSigned && BitNo == SrcSize - 1)
12109 return false;
12110
12111 ExpectedValue <<= BitNo;
12112
12113 bool IsReversedCC = false;
12114 if (CmpValue != ExpectedValue) {
12115 if (!IsReversible)
12116 return false;
12117 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
12118 if (!IsReversedCC)
12119 return false;
12120 }
12121
12122 Register DefReg = Def->getOperand(0).getReg();
12123 if (IsReversedCC && !MRI->hasOneNonDBGUse(DefReg))
12124 return false;
12125
12126 if (!optimizeSCC(Def, &CmpInstr, /*NeedInversion=*/false))
12127 return false;
12128
12129 if (!MRI->use_nodbg_empty(DefReg)) {
12130 assert(!IsReversedCC);
12131 return true;
12132 }
12133
12134 // Replace AND with unused result with a S_BITCMP.
12135 MachineBasicBlock *MBB = Def->getParent();
12136
12137 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
12138 : AMDGPU::S_BITCMP1_B32
12139 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
12140 : AMDGPU::S_BITCMP1_B64;
12141
12142 BuildMI(*MBB, Def, Def->getDebugLoc(), get(NewOpc))
12143 .add(*SrcOp)
12144 .addImm(BitNo);
12145 Def->eraseFromParent();
12146
12147 return true;
12148 };
12149
12150 switch (CmpInstr.getOpcode()) {
12151 default:
12152 break;
12153 case AMDGPU::S_CMP_EQ_U32:
12154 case AMDGPU::S_CMP_EQ_I32:
12155 case AMDGPU::S_CMPK_EQ_U32:
12156 case AMDGPU::S_CMPK_EQ_I32:
12157 return optimizeCmpAnd(1, 32, true, false) ||
12158 optimizeCmpSelect(/*NeedInversion=*/true);
12159 case AMDGPU::S_CMP_GE_U32:
12160 case AMDGPU::S_CMPK_GE_U32:
12161 return optimizeCmpAnd(1, 32, false, false);
12162 case AMDGPU::S_CMP_GE_I32:
12163 case AMDGPU::S_CMPK_GE_I32:
12164 return optimizeCmpAnd(1, 32, false, true);
12165 case AMDGPU::S_CMP_EQ_U64:
12166 return optimizeCmpAnd(1, 64, true, false) ||
12167 optimizeCmpSelect(/*NeedInversion=*/true);
12168 case AMDGPU::S_CMP_LG_U32:
12169 case AMDGPU::S_CMP_LG_I32:
12170 case AMDGPU::S_CMPK_LG_U32:
12171 case AMDGPU::S_CMPK_LG_I32:
12172 return optimizeCmpAnd(0, 32, true, false) ||
12173 optimizeCmpSelect(/*NeedInversion=*/false);
12174 case AMDGPU::S_CMP_GT_U32:
12175 case AMDGPU::S_CMPK_GT_U32:
12176 return optimizeCmpAnd(0, 32, false, false);
12177 case AMDGPU::S_CMP_GT_I32:
12178 case AMDGPU::S_CMPK_GT_I32:
12179 return optimizeCmpAnd(0, 32, false, true);
12180 case AMDGPU::S_CMP_LG_U64:
12181 return optimizeCmpAnd(0, 64, true, false) ||
12182 optimizeCmpSelect(/*NeedInversion=*/false);
12183 }
12184
12185 return false;
12186}
12187
12189 AMDGPU::OpName OpName) const {
12190 if (!ST.needsAlignedVGPRs())
12191 return;
12192
12193 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
12194 if (OpNo < 0)
12195 return;
12196 MachineOperand &Op = MI.getOperand(OpNo);
12197 if (getOpSize(MI, OpNo) > 4)
12198 return;
12199
12200 // Add implicit aligned super-reg to force alignment on the data operand.
12201 const DebugLoc &DL = MI.getDebugLoc();
12202 MachineBasicBlock *BB = MI.getParent();
12204 Register DataReg = Op.getReg();
12205 bool IsAGPR = RI.isAGPR(MRI, DataReg);
12206 Register Undef = MRI.createVirtualRegister(
12207 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
12208 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);
12209 Register NewVR =
12210 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
12211 : &AMDGPU::VReg_64_Align2RegClass);
12212 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR)
12213 .addReg(DataReg, {}, Op.getSubReg())
12214 .addImm(AMDGPU::sub0)
12215 .addReg(Undef)
12216 .addImm(AMDGPU::sub1);
12217 Op.setReg(NewVR);
12218 Op.setSubReg(AMDGPU::sub0);
12219 MI.addOperand(MachineOperand::CreateReg(NewVR, false, true));
12220}
12221
12223 if (!SchedModel.hasInstrSchedModel())
12224 return 0;
12225
12226 // The repeat rate is the throughput-limiting resource occupancy: the largest
12227 // number of cycles any written processor resource is held.
12228 const MCSchedClassDesc *SCDesc = SchedModel.resolveSchedClass(&MI);
12229 unsigned RepeatRate = 0;
12231 PI = SchedModel.getWriteProcResBegin(SCDesc),
12232 PE = SchedModel.getWriteProcResEnd(SCDesc);
12233 PI != PE; ++PI) {
12234 RepeatRate = std::max(RepeatRate, (unsigned)PI->ReleaseAtCycle);
12235 }
12236
12237 return RepeatRate;
12238}
12239
12241 if (isIGLP(*MI))
12242 return false;
12243
12245}
12246
12248 if (!isWMMA(MI) && !isSWMMAC(MI))
12249 return false;
12250
12251 if (ST.hasGFX1250Insts())
12252 return AMDGPU::getWMMAIsXDL(MI.getOpcode());
12253
12254 return true;
12255}
12256
12258 unsigned Opcode = MI.getOpcode();
12259
12260 if (AMDGPU::isGFX12Plus(ST))
12261 return isDOT(MI) || isXDLWMMA(MI);
12262
12263 if (!isMAI(MI) || isDGEMM(Opcode) ||
12264 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
12265 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
12266 return false;
12267
12268 if (!ST.hasGFX940Insts())
12269 return true;
12270
12271 return AMDGPU::getMAIIsGFX940XDL(Opcode);
12272}
12273
12275 constexpr unsigned MinOrderedXDLVAVDSTVGPRs = 8;
12276 if (!AMDGPU::isGFX1250(ST) || !isXDL(MI))
12277 return false;
12278
12279 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
12280 const TargetRegisterClass *DstRC =
12281 getRegClass(MI.getDesc(), Dst->getOperandNo());
12282 return DstRC && RI.isVGPRClass(DstRC) &&
12283 RI.getRegSizeInBits(*DstRC) >= MinOrderedXDLVAVDSTVGPRs * 32;
12284}
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Shared AMDGPU asyncmark stage definitions.
uint64_t uint64_t
uint64_t Imm
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
AMDGPU Register Bank Select
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
static Register UseReg(const MachineOperand &MO)
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
static bool isUndef(const MachineInstr &MI)
TargetInstrInfo::RegSubRegPair RegSubRegPair
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
uint64_t High
uint64_t IntrinsicInst * II
#define P(N)
R600 Clause Merge
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
This file declares the machine register scavenger class.
static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RC, bool Forward)
static std::optional< std::pair< int64_t, int64_t > > getSelectConstants(const SIInstrInfo &TII, const MachineRegisterInfo &MRI, const MachineInstr &Sel)
If Sel is an S_CSELECT* of two different constants A and B, return them, truncated to the width of th...
static void expandLoadStoreIdx(const SIInstrInfo &TII, MachineInstr &MI)
static void lowerVGPRCopyToM0(const SIInstrInfo &TII, MachineInstr &Copy, ArrayRef< MachineInstr * > Readers, MachineRegisterInfo &MRI, MachineDominatorTree *MDT)
static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc)
static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize)
static bool compareMachineOp(const MachineOperand &Op0, const MachineOperand &Op1)
static bool isStride64(unsigned Opc)
static MachineBasicBlock * generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI, ArrayRef< MachineOperand * > ScalarOps, MachineDominatorTree *MDT, MachineBasicBlock::iterator Begin=nullptr, MachineBasicBlock::iterator End=nullptr, ArrayRef< Register > PhySGPRs={})
#define GENERATE_RENAMED_GFX9_CASES(OPCODE)
static std::tuple< unsigned, unsigned > extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc)
static unsigned VOP3OpIdxToSrcN(const MachineInstr &MI, unsigned OpIdx)
static bool followSubRegDef(MachineInstr &MI, TargetInstrInfo::RegSubRegPair &RSR)
static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize)
static MachineInstr * swapImmOperands(MachineInstr &MI, MachineOperand &NonRegOp1, MachineOperand &NonRegOp2)
static void copyFlagsToImplicitVCC(MachineInstr &MI, const MachineOperand &Orig)
static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA, LocationSize WidthB, int OffsetB)
static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, RegScavenger &RS, bool RegsOverlap, Register ImpUseSuperReg=Register())
Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908.
static unsigned getWWMRegSpillSaveOpcode(unsigned Size, bool IsVectorSuperClass)
static bool memOpsHaveSameBaseOperands(ArrayRef< const MachineOperand * > BaseOps1, ArrayRef< const MachineOperand * > BaseOps2)
static unsigned getWWMRegSpillRestoreOpcode(unsigned Size, bool IsVectorSuperClass)
static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI)
static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion, unsigned &NewDefOpc)
static bool isSCCDeadOnExit(MachineBasicBlock *MBB)
static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize)
static unsigned subtargetEncodingFamily(const GCNSubtarget &ST)
static void preserveCondRegFlags(MachineOperand &CondReg, const MachineOperand &OrigCond)
static Register findImplicitSGPRRead(const MachineInstr &MI)
static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc)
static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI)
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef< const MachineOperand * > BaseOps1, const MachineInstr &MI2, ArrayRef< const MachineOperand * > BaseOps2)
static unsigned getSGPRSpillRestoreOpcode(unsigned Size)
static bool isRegOrFI(const MachineOperand &MO)
static bool isVCmp(const SIInstrInfo &TII, const MachineInstr &MI)
Return true if MI is a VALU comparison, i.e.
static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI)
static constexpr AMDGPU::OpName ModifierOpNames[]
static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, const char *Msg="illegal VGPR to SGPR copy")
static MachineInstr * swapRegAndNonRegOperand(MachineInstr &MI, MachineOperand &RegOp, MachineOperand &NonRegOp)
static bool isLegalPkF32OpSel(const SIInstrInfo &TII, const MachineInstr &MI, bool CheckSwappedSrc=false)
static bool shouldReadExec(const MachineInstr &MI)
static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc)
static bool isRenamedInGFX9(int Opcode)
static TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd)
static bool changesVGPRIndexingMode(const MachineInstr &MI)
static bool isSubRegOf(const SIRegisterInfo &TRI, const MachineOperand &SuperVec, const MachineOperand &SubReg)
static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1, AMDGPU::OpName OpName)
Returns true if both nodes have the same value for the given operand Op, or if both nodes do not have...
static std::pair< unsigned, unsigned > splitGlobalAddressRelocFlags(const MachineOperand &SrcOp)
static unsigned getNumOperandsNoGlue(SDNode *Node)
static bool canRemat(const MachineInstr &MI)
static unsigned getAVSpillRestoreOpcode(unsigned Size)
static void emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB, MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL, ArrayRef< MachineOperand * > ScalarOps, ArrayRef< Register > PhySGPRs={})
static unsigned getVGPRSpillRestoreOpcode(unsigned Size)
Interface definition for SIInstrInfo.
bool IsDead
const char * Msg
This file contains some templates that are useful if you are working with the STL at all.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
#define LLVM_DEBUG(...)
Definition Debug.h:119
static const LaneMaskConstants & get(const GCNSubtarget &ST)
static LLVM_ABI Semantics SemanticsToEnum(const llvm::fltSemantics &Sem)
Definition APFloat.cpp:185
Class for arbitrary precision integers.
Definition APInt.h:78
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
const T & front() const
Get the first element.
Definition ArrayRef.h:144
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
uint64_t getZExtValue() const
Opaque handle to a cycle within a GenericCycleInfo that wraps the cycle's preorder index.
A debug info location.
Definition DebugLoc.h:126
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
Diagnostic information for unsupported feature in backend.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
void getExitingBlocks(CycleRef C, SmallVectorImpl< BlockT * > &TmpStorage) const
Return all blocks of C that have a successor outside of C.
CycleRef getParentCycle(CycleRef C) const
bool contains(CycleRef Outer, CycleRef Inner) const
Returns true iff Outer contains Inner. O(1). Non-strict.
CycleRef getCycle(const BlockT *Block) const
Find the innermost cycle containing Block.
Itinerary data supplied by a subtarget to be used by a target.
constexpr unsigned getAddressSpace() const
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
A live range for subregisters.
LiveInterval - This class represents the liveness of a register, or stack slot.
iterator_range< subrange_iterator > subranges()
bool hasInterval(Register Reg) const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveInterval & getInterval(Register Reg)
LLVM_ABI bool shrinkToUses(LiveInterval *li, SmallVectorImpl< MachineInstr * > *dead=nullptr, LaneBitmask SubRangeMask=LaneBitmask::getAll())
After removing some uses of a register, shrink its live range to just the remaining uses.
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
This class represents the liveness of a register, stack slot, etc.
bool hasValue() const
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
static const MCBinaryExpr * createAnd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition MCExpr.h:347
static const MCBinaryExpr * createAShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition MCExpr.h:417
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition MCExpr.h:427
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition MCExpr.cpp:212
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
unsigned getSize() const
Return the number of bytes in the encoding of this instruction, or zero if the encoding size cannot b...
ArrayRef< MCPhysReg > implicit_uses() const
Return a list of registers that are potentially read by any instance of this machine instruction.
unsigned getOpcode() const
Return the opcode number for this descriptor.
This holds information about one operand of a machine instruction, indicating the register class for ...
Definition MCInstrDesc.h:88
uint8_t OperandType
Information about the type of the operand.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition MCInstrDesc.h:94
bool hasSuperClassEq(const MCRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx, SMLoc Loc=SMLoc())
Definition MCExpr.h:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
LLVM_ABI void setVariableValue(const MCExpr *Value)
Definition MCSymbol.cpp:51
Helper class for constructing bundles of MachineInstrs.
MachineBasicBlock::instr_iterator begin() const
Return an iterator to the first bundled instruction.
MIBundleBuilder & append(MachineInstr *MI)
Insert MI into MBB by appending it to the instructions in the bundle.
MIRFormater - Interface to format MIR operand based on target.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
LLVM_ABI LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI, MCRegister Reg, const_iterator Before, unsigned Neighborhood=10) const
Return whether (physical) register Reg has been defined and not killed as of just before Before.
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
Instructions::const_iterator const_instr_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
@ LQR_Dead
Register is known to be fully dead.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isImmutableObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to an immutable object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
bool isCopy() const
const MachineBasicBlock * getParent() const
LLVM_ABI void addImplicitDefUseOperands(MachineFunction &MF)
Add all implicit def and use operands to this instruction.
bool isBundle() const
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
mop_range implicit_operands()
bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
LLVM_ABI bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
void untieRegOperand(unsigned OpIdx)
Break any tie involving OpIdx.
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI void eraseFromBundle()
Unlink 'this' from its basic block and delete it.
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
mop_range explicit_operands()
LLVM_ABI void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
bool isMoveImmediate(QueryType Type=IgnoreBundle) const
Return true if this instruction is a move immediate (including conditional moves) instruction.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
filtered_mop_range all_uses()
Returns an iterator range over all operands that are (explicit or implicit) register uses.
LLVM_ABI void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol)
Set a symbol that will be emitted just after the instruction itself.
LLVM_ABI void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo)
Clear all kill flags affecting Reg.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
MachineOperand * findRegisterDefOperand(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
LLVM_ABI unsigned getOperandNo() const
Returns the index of this operand in the instruction that it belongs to.
const GlobalValue * getGlobal() const
LLVM_ABI void ChangeToFrameIndex(int Idx, unsigned TargetFlags=0)
Replace this operand with a frame index.
void setImm(int64_t immVal)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
LLVM_ABI void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
LLVM_ABI void ChangeToGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
ChangeToGA - Replace this operand with a new global address operand.
void setIsKill(bool Val=true)
LLVM_ABI void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value.
void setOffset(int64_t Offset)
unsigned getTargetFlags() const
static MachineOperand CreateImm(int64_t Val)
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
bool isEarlyClobber() const
Register getReg() const
getReg - Returns the register number.
bool isTargetIndex() const
isTargetIndex - Tests if this is a MO_TargetIndex operand.
void setTargetFlags(unsigned F)
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
@ MO_Immediate
Immediate operand.
@ MO_Register
Register operand.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
int64_t getOffset() const
Return the offset from the symbol in this operand.
bool isFPImm() const
isFPImm - Tests if this is a MO_FPImmediate operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
iterator_range< use_nodbg_iterator > use_nodbg_operands(Register Reg) const
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
const MachineFunction & getMF() const
void setSimpleHint(Register VReg, Register PrefReg)
Specify the preferred (target independent) register allocation hint for the specified virtual registe...
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
iterator_range< use_iterator > use_operands(Register Reg) const
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition Register.h:107
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isLegalMUBUFImmOffset(unsigned Imm) const
bool isInlineConstant(const APInt &Imm) const
void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const
Fix operands in MI to satisfy constant bus requirements.
bool canAddToBBProlog(const MachineInstr &MI) const
static std::optional< unsigned > extractSubregFromGlobalAddressFlags(unsigned TargetFlags, unsigned SubRegIndex)
Return the target flags for a global address operand reading subregister SubRegIndex of a 64-bit glob...
static bool isDS(const MachineInstr &MI)
MachineBasicBlock * legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT=nullptr) const
Legalize all operands in this instruction.
bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0, int64_t &Offset1) const override
unsigned getLiveRangeSplitOpcode(Register Reg, const MachineFunction &MF) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
static bool isNeverUniform(const MachineInstr &MI)
bool isXDLWMMA(const MachineInstr &MI) const
void handleCopyToPhysHelper(SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst, MachineRegisterInfo &MRI, MachineDominatorTree *MDT, DenseMap< MachineInstr *, V2PhysSCopyInfo > &WaterFalls, DenseMap< MachineInstr *, bool > &V2SPhyCopiesToErase) const
bool isBasicBlockPrologue(const MachineInstr &MI, Register Reg=Register()) const override
uint64_t getDefaultRsrcDataFormat() const
static bool isSOPP(const MachineInstr &MI)
bool mayAccessScratch(const MachineInstr &MI) const
bool isIGLP(unsigned Opcode) const
static bool isFLATScratch(const MachineInstr &MI)
Register buildVGPR32FromLo16(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register Src, unsigned SrcSubReg) const
Build a 32-bit VGPR whose low half is the 16-bit value Src.SrcSubReg and whose high half is undefined...
bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, AMDGPU::FlatAddrSpace FlatVariant) const
Returns if Offset is legal for the subtarget as the offset to a FLAT encoded instruction with the giv...
const MCInstrDesc & getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, bool IsSGPR) const
MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DestReg) const
Return a partially built integer add instruction without carry.
bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0, int64_t Offset1, unsigned NumLoads) const override
bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, Align Alignment=Align(4)) const
bool isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const
Replace the instructions opcode with the equivalent VALU opcode.
static bool isSMRD(const MachineInstr &MI)
unsigned getGFX1250BlockingCyclesTable(const MachineInstr &MI) const
GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register Reg, SlotIndexes *Indexes=nullptr) const
void storeRegToStackSlotCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC) const
bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineOperand &MO, const MCOperandInfo &OpInfo) const
Returns true if this operand uses the constant bus.
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
static unsigned getFoldableCopySrcIdx(const MachineInstr &MI)
unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const
Return the size in bytes of the operand OpNo on the given.
void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &LdSt, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const final
static std::optional< int64_t > extractSubregFromImm(int64_t ImmVal, unsigned SubRegIndex)
Return the extracted immediate value in a subregister use from a constant materialized in a super reg...
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isMTBUF(const MachineInstr &MI)
const MCInstrDesc & getIndirectGPRIDXPseudo(unsigned VecSize, bool IsIndirectSrc) const
static bool isDGEMM(unsigned Opcode)
static bool isEXP(const MachineInstr &MI)
static bool isSALU(const MachineInstr &MI)
static bool setsSCCIfResultIsNonZero(const MachineInstr &MI)
const MIRFormatter * getMIRFormatter() const override
static bool isXcntDrain(const MachineInstr &MI)
True if MI implicitly drains XCNT.
void legalizeGenericOperand(MachineBasicBlock &InsertMBB, MachineBasicBlock::iterator I, const TargetRegisterClass *DstRC, MachineOperand &Op, MachineRegisterInfo &MRI, const DebugLoc &DL) const
MachineInstr * buildShrunkInst(MachineInstr &MI, unsigned NewOpcode) const
static bool isVOP2(const MachineInstr &MI)
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
static bool isSDWA(const MachineInstr &MI)
const MCInstrDesc & getKillTerminatorFromPseudo(unsigned Opcode) const
void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned Quantity) const override
static bool isGather4(const MachineInstr &MI)
MachineInstr * getWholeWaveFunctionSetup(MachineFunction &MF) const
static bool isDOT(const MachineInstr &MI)
std::unique_ptr< PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
InstSizeVerifyMode getInstSizeVerifyMode(const MachineInstr &MI) const override
MachineInstr * createPHISourceCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const override
static bool usesTENSOR_CNT(const MachineInstr &MI)
bool hasModifiers(unsigned Opcode) const
Return true if this instruction has any modifiers.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
static bool isSWMMAC(const MachineInstr &MI)
ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *II, const ScheduleDAGMI *DAG) const override
bool isWave32() const
bool isHighLatencyDef(int Opc) const override
void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const
Legalize the OpIndex operand of this instruction by inserting a MOV.
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isVOPC(const MachineInstr &MI)
void removeModOperands(MachineInstr &MI) const
unsigned getRepeatRate(const MachineInstr &MI) const
Get the repeat rate for a VALU instruction from the scheduling model.
unsigned getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC, unsigned Size, const SIMachineFunctionInfo &MFI) const
bool isLegalSingleSGPRReadInstOperand(const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN, const MachineOperand *MO=nullptr) const
Check if MO would be a legal operand for a single-SGPR-read instruction.
bool isXDL(const MachineInstr &MI) const
Register isStackAccess(const MachineInstr &MI, int &FrameIndex, TypeSize &MemBytes) const
static bool isVIMAGE(const MachineInstr &MI)
void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const
static bool isSOP2(const MachineInstr &MI)
static bool isGWS(const MachineInstr &MI)
bool hasRAWDependency(const MachineInstr &FirstMI, const MachineInstr &SecondMI) const
bool isLegalAV64PseudoImm(uint64_t Imm) const
Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
bool isNeverCoissue(MachineInstr &MI) const
static bool isBUF(const MachineInstr &MI)
bool isNonCommutableDPP(const MachineInstr &MI) const
bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const
bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx, unsigned toIdx) const
static bool isFLATGlobal(const MachineInstr &MI)
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isGlobalMemoryObject(const MachineInstr *MI) const override
bool canInsertSelect(const MachineBasicBlock &MBB, ArrayRef< MachineOperand > Cond, Register DstReg, Register TrueReg, Register FalseReg, SelectExpansion &Exp) const override
static bool isVSAMPLE(const MachineInstr &MI)
bool isBufferSMRD(const MachineInstr &MI) const
static bool isKillTerminator(unsigned Opcode)
bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const
If OpX is multicycle, anti-dependencies are not allowed.
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0, unsigned &SrcOpIdx1) const override
void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register Reg, bool IsSCCLive, SlotIndexes *Indexes=nullptr) const
bool hasVALU32BitEncoding(unsigned Opcode) const
Return true if this 64-bit VALU instruction has a 32-bit encoding.
unsigned getBlockingCycles(const MachineInstr &MI) const
unsigned getMovOpcode(const TargetRegisterClass *DstRC) const
Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex, TypeSize &MemBytes) const
unsigned buildExtractSubReg(MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, unsigned SubIdx, const TargetRegisterClass *SubRC) const
void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const
Legalize operands in MI by either commuting it or inserting a copy of src1.
static bool isVPermPk16(unsigned Opcode)
static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA)
bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const final
static bool isTRANS(const MachineInstr &MI)
static bool isImage(const MachineInstr &MI)
static bool isSOPK(const MachineInstr &MI)
const TargetRegisterClass * getOpRegClass(const MachineInstr &MI, unsigned OpNo) const
Return the correct register class for OpNo.
MachineBasicBlock * insertSimulatedTrap(MachineRegisterInfo &MRI, MachineBasicBlock &MBB, MachineInstr &MI, const DebugLoc &DL) const
Build instructions that simulate the behavior of a s_trap 2 instructions for hardware (namely,...
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static unsigned getDSShaderTypeValue(const MachineFunction &MF)
static bool isFoldableCopy(const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
bool expandPostRAPseudo(MachineInstr &MI) const override
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT, ArrayRef< MachineOperand * > ScalarOps, ArrayRef< Register > PhySGPRs={}) const
Wrapper function for generating waterfall for instruction MI This function take into consideration of...
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, Register VReg, unsigned SubReg=0, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
AMDGPU::AsyncStages getAsyncStage(const MachineInstr &MI) const
static bool isSegmentSpecificFLAT(const MachineInstr &MI)
bool isReMaterializableImpl(const MachineInstr &MI) const override
static bool isVOP3(const MCInstrDesc &Desc)
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool physRegUsesConstantBus(const MachineOperand &Reg) const
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const
static bool isDPP(const MachineInstr &MI)
bool analyzeBranchImpl(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const
static bool isMFMA(const MachineInstr &MI)
bool isLowLatencyInstruction(const MachineInstr &MI) const
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is a instruction that moves/copies value from one register to ano...
void mutateAndCleanupImplicit(MachineInstr &MI, const MCInstrDesc &NewDesc) const
ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const
static bool isMAI(const MCInstrDesc &Desc)
static bool isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode)
void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, LaneBitmask UsedLanes=LaneBitmask::getAll()) const override
static bool usesLGKM_CNT(const MachineInstr &MI)
void legalizeOperandsVALUt16(MachineInstr &Inst, MachineRegisterInfo &MRI) const
Fix operands in Inst to fix 16bit SALU to VALU lowering.
bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, const MachineOperand &MO) const
bool canShrink(const MachineInstr &MI, const MachineRegisterInfo &MRI) const
const MachineOperand & getCalleeOperand(const MachineInstr &MI) const override
bool isAsmOnlyOpcode(int MCOp) const
Check if this instruction should only be used by assembler.
bool isAlwaysGDS(uint32_t Opcode) const
static bool isVGPRSpill(const MachineInstr &MI)
ScheduleHazardRecognizer * CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const override
This is used by the post-RA scheduler (SchedulePostRAList.cpp).
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
unsigned getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr &MI, unsigned *PredCost=nullptr) const override
unsigned getVectorRegSpillSaveOpcode(Register Reg, const TargetRegisterClass *RC, unsigned Size, const SIMachineFunctionInfo &MFI, bool NeedsCFI) const
int64_t getNamedImmOperand(const MachineInstr &MI, AMDGPU::OpName OperandName) const
Get required immediate operand.
ArrayRef< std::pair< int, const char * > > getSerializableTargetIndices() const override
bool regUsesConstantBus(const MachineOperand &Reg, const MachineRegisterInfo &MRI) const
static bool isMIMG(const MachineInstr &MI)
MachineOperand buildExtractSubRegOrImm(MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, unsigned SubIdx, const TargetRegisterClass *SubRC) const
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
bool isLegalRegOperand(const MachineRegisterInfo &MRI, const MCOperandInfo &OpInfo, const MachineOperand &MO) const
Check if MO (a register operand) is a legal register for the given operand description or operand ind...
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
bool isVAVDSTOrderedXDL(const MachineInstr &MI) const
On gfx1250, an XDL instruction whose destination spans at least eight VGPRs decrements VA_VDST only a...
unsigned getVALUOp(const MachineInstr &MI) const
static bool modifiesModeRegister(const MachineInstr &MI)
Return true if the instruction modifies the mode register.q.
Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI, const TargetRegisterClass *DstRC=nullptr) const
Copy a value from a VGPR (SrcReg) to SGPR.
bool hasDivergentBranch(const MachineBasicBlock *MBB) const
Return whether the block terminate with divergent branch.
std::pair< int64_t, int64_t > splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, AMDGPU::FlatAddrSpace FlatVariant) const
Split COffsetVal into {immediate offset field, remainder offset} values.
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
void fixImplicitOperands(MachineInstr &MI) const
bool moveFlatAddrToVGPR(MachineInstr &Inst) const
Change SADDR form of a FLAT Inst to its VADDR form if saddr operand was moved to VGPR.
static bool usesASYNC_CNT(const MachineInstr &MI)
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DestReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool isMaskedByExec(Register Reg, const MachineInstr &Use, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Return true if Reg is a lane mask that already has 0 in every bit corresponding to a lane that is ina...
void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const
bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0, AMDGPU::OpName Src0OpName, MachineOperand &Src1, AMDGPU::OpName Src1OpName) const
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
bool isAsync(const MachineInstr &MI) const
bool memOpsHaveSameBase(ArrayRef< const MachineOperand * > BaseOps1, ArrayRef< const MachineOperand * > BaseOps2) const
Return true if the memory operations described by BaseOps1 and BaseOps2 (as returned by getMemOperand...
bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const
This function is used to determine if an instruction can be safely executed under EXEC = 0 without ha...
bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg, int64_t &ImmVal) const override
static bool isAtomic(const MachineInstr &MI)
bool isLiteralOperandLegal(const MCInstrDesc &InstDesc, const MCOperandInfo &OpInfo) const
static bool isWWMRegSpillOpcode(uint32_t Opcode)
static bool sopkIsZext(unsigned Opcode)
static bool isSGPRSpill(const MachineInstr &MI)
static bool isWMMA(const MachineInstr &MI)
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const
Returns true if the instruction could potentially depend on the value of exec.
void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
std::pair< MachineInstr *, MachineInstr * > expandMovDPP64(MachineInstr &MI) const
static bool isSOPC(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
bool isBarrier(unsigned Opcode) const
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx0, unsigned OpIdx1) const override
bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const
int pseudoToMCOpcode(int Opcode) const
Return a target-specific opcode if Opcode is a pseudo instruction.
const MCInstrDesc & getMCOpcodeFromPseudo(unsigned Opcode) const
Return the descriptor of the target-specific machine instruction that corresponds to the specified ps...
static bool usesVM_CNT(const MachineInstr &MI)
MachineInstr * createPHIDestinationCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, Register Dst) const override
static bool isFixedSize(const MachineInstr &MI)
bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo, MachineCycleInfo *CI) const override
LLVM_READONLY int commuteOpcode(unsigned Opc) const
ValueUniformity getValueUniformity(const MachineInstr &MI) const final
uint64_t getScratchRsrcWords23() const
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, AMDGPU::OpName OperandName) const
Returns the operand named Op.
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveIntervals *LIS) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx, const MachineOperand *MO=nullptr) const
Check if MO is a legal operand if it was the OpIdx Operand for MI.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const
Returns true if negative offsets are allowed for the given FlatVariant.
void moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst, DenseMap< MachineInstr *, V2PhysSCopyInfo > &WaterFalls, DenseMap< MachineInstr *, bool > &V2SPhyCopiesToErase) const
static bool isLDSDMA(const MachineInstr &MI)
static bool isVOP1(const MachineInstr &MI)
SIInstrInfo(const GCNSubtarget &ST)
std::optional< int64_t > getImmOrMaterializedImm(const MachineRegisterInfo &MRI, const MachineOperand &Op, MachineInstr **DefMI=nullptr) const
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool hasAnyModifiersSet(const MachineInstr &MI) const
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void setHasSpilledVGPRs(bool Spill=true)
bool isWWMReg(Register Reg) const
bool checkFlag(Register Reg, uint8_t Flag) const
void setHasSpilledSGPRs(bool Spill=true)
unsigned getScratchReservedForDynamicVGPRs() const
ArrayRef< int16_t > getRegSplitParts(const TargetRegisterClass *RC, unsigned EltSize) const
unsigned getHWRegIndex(MCRegister Reg) const
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
bool isSGPRReg(const MachineRegisterInfo &MRI, Register Reg) const
unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const override
unsigned getChannelFromSubReg(unsigned SubReg) const
static bool isSGPRClass(const TargetRegisterClass *RC)
static bool isAGPRClass(const TargetRegisterClass *RC)
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
virtual bool hasVRegLiveness() const
Return true if this DAG supports VReg liveness and RegPressure.
MachineFunction & MF
Machine function.
HazardRecognizer - This determines whether or not an instruction can be issued this cycle,...
SlotIndex - An opaque wrapper around machine indexes.
Definition SlotIndexes.h:70
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
SlotIndexes pass.
SlotIndex insertMachineInstrInMaps(MachineInstr &MI, bool Late=false)
Insert the given machine instruction into the mapping.
Implements a dense probed hash-table based set with some number of buckets stored inline.
Definition DenseSet.h:295
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
int64_t getImm() const
Register getReg() const
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Object returned by analyzeLoopForPipelining.
virtual ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *, const ScheduleDAGMI *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
virtual MachineInstr * createPHIDestinationCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, Register Dst) const
During PHI eleimination lets target to make necessary checks and insert the copy to the PHI destinati...
virtual const MachineOperand & getCalleeOperand(const MachineInstr &MI) const
Returns the callee operand from the given MI.
virtual void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, LaneBitmask UsedLanes=LaneBitmask::getAll()) const
Re-issue the specified 'original' instruction at the specific location targeting a new destination re...
virtual MachineInstr * createPHISourceCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const
During PHI eleimination lets target to make necessary checks and insert the copy to the PHI destinati...
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool isGlobalMemoryObject(const MachineInstr *MI) const
Returns true if MI is an instruction we are unable to reason about (like a call or something with unm...
virtual bool expandPostRAPseudo(MachineInstr &MI) const
This function is called for all pseudo instructions that remain after register allocation.
const MCAsmInfo & getMCAsmInfo() const
Return target specific asm information.
const MCWriteProcResEntry * ProcResIter
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition DenseSet.h:188
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ PRIVATE_ADDRESS
Address space for private memory.
unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst)
const uint64_t RSRC_DATA_FORMAT
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool getWMMAIsXDL(unsigned Opc)
unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc)
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isDPMACCInstruction(unsigned Opc)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_READONLY int32_t getCommuteRev(uint32_t Opcode)
LLVM_READONLY int32_t getCommuteOrig(uint32_t Opcode)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool isGFX12Plus(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
constexpr bool isValidAsyncStageMask(int64_t Mask)
AsyncStages
Bit mask of the async stages tracked by the asyncmark / wait_asyncmark intrinsics.
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
LLVM_READONLY int32_t getGlobalVaddrOp(uint32_t Opcode)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
LLVM_READONLY int32_t getMFMAEarlyClobberOp(uint32_t Opcode)
bool getMAIIsGFX940XDL(unsigned Opc)
const uint64_t RSRC_ELEMENT_SIZE_SHIFT
bool isIntrinsicAlwaysUniform(unsigned IntrID)
bool isPackedSingleSGPR64BitInst(unsigned Opc)
The opcode is a packed 64-bit instruction which only reads low 64 bits of a scalar operand and propag...
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
LLVM_READONLY int32_t getIfAddr64Inst(uint32_t Opcode)
Check if Opcode is an Addr64 opcode.
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
const uint64_t RSRC_TID_ENABLE
LLVM_READONLY int32_t getVOPe32(uint32_t Opcode)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
bool isGenericAtomic(unsigned Opc)
LLVM_READNONE bool isInlinableIntLiteral(int64_t Literal)
Is this literal inlinable, and not one of the values intended for floating point values.
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
LLVM_READONLY int32_t getAddr64Inst(uint32_t Opcode)
int32_t getMCOpcode(uint32_t Opcode, unsigned Gen)
@ OPERAND_REG_IMM_V2FP64
Definition SIDefines.h:447
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
Definition SIDefines.h:465
@ OPERAND_REG_IMM_INT64
Definition SIDefines.h:432
@ OPERAND_REG_IMM_V2FP16
Definition SIDefines.h:440
@ OPERAND_REG_INLINE_C_FP64
Definition SIDefines.h:456
@ OPERAND_REG_IMM_NOINLINE_FP16
Definition SIDefines.h:438
@ OPERAND_REG_INLINE_C_BF16
Definition SIDefines.h:453
@ OPERAND_REG_INLINE_C_V2BF16
Definition SIDefines.h:458
@ OPERAND_REG_IMM_V2INT64
Definition SIDefines.h:443
@ OPERAND_REG_IMM_V2INT16
Definition SIDefines.h:442
@ OPERAND_REG_IMM_BF16
Definition SIDefines.h:436
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
Definition SIDefines.h:431
@ OPERAND_REG_IMM_V2BF16
Definition SIDefines.h:439
@ OPERAND_REG_IMM_FP16
Definition SIDefines.h:437
@ OPERAND_REG_IMM_V2FP16_SPLAT
Definition SIDefines.h:441
@ OPERAND_REG_INLINE_C_INT64
Definition SIDefines.h:452
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
Definition SIDefines.h:450
@ OPERAND_REG_IMM_NOINLINE_V2FP16
Definition SIDefines.h:444
@ OPERAND_REG_IMM_FP64
Definition SIDefines.h:435
@ OPERAND_REG_INLINE_C_V2FP16
Definition SIDefines.h:459
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
Definition SIDefines.h:470
@ OPERAND_REG_INLINE_AC_FP32
Definition SIDefines.h:471
@ OPERAND_REG_IMM_V2INT32
Definition SIDefines.h:445
@ OPERAND_SDWA_VOPC_DST
Definition SIDefines.h:482
@ OPERAND_REG_IMM_FP32
Definition SIDefines.h:434
@ OPERAND_REG_INLINE_C_FP32
Definition SIDefines.h:455
@ OPERAND_REG_INLINE_C_INT32
Definition SIDefines.h:451
@ OPERAND_REG_INLINE_C_V2INT16
Definition SIDefines.h:457
@ OPERAND_INLINE_C_AV64_PSEUDO
Definition SIDefines.h:476
@ OPERAND_REG_IMM_V2FP32
Definition SIDefines.h:446
@ OPERAND_REG_INLINE_AC_FP64
Definition SIDefines.h:472
@ OPERAND_REG_INLINE_C_FP16
Definition SIDefines.h:454
@ OPERAND_REG_IMM_INT16
Definition SIDefines.h:433
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
Definition SIDefines.h:462
LLVM_READONLY int32_t getBasicFromSDWAOp(uint32_t Opcode)
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
@ TI_SCRATCH_RSRC_DWORD1
Definition AMDGPU.h:677
@ TI_SCRATCH_RSRC_DWORD3
Definition AMDGPU.h:679
@ TI_SCRATCH_RSRC_DWORD0
Definition AMDGPU.h:676
@ TI_SCRATCH_RSRC_DWORD2
Definition AMDGPU.h:678
@ TI_CONSTDATA_START
Definition AMDGPU.h:675
bool isGFX1250(const MCSubtargetInfo &STI)
bool isSingleSGPRReadInst(unsigned Opc)
Packed instructions that read a single SGPR for SGPR operands, except for 64-bit elements which read ...
bool isExtendedGlobalAddrSpace(unsigned AS)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
const uint64_t RSRC_INDEX_STRIDE_SHIFT
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
LLVM_READONLY int32_t getFlatScratchInstSVfromSS(uint32_t Opcode)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isGraphics(CallingConv::ID CC)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ OPERAND_GENERIC_4
Definition MCInstrDesc.h:71
@ OPERAND_GENERIC_2
Definition MCInstrDesc.h:69
@ OPERAND_GENERIC_1
Definition MCInstrDesc.h:68
@ OPERAND_GENERIC_3
Definition MCInstrDesc.h:70
@ OPERAND_IMMEDIATE
Definition MCInstrDesc.h:61
@ OPERAND_GENERIC_0
Definition MCInstrDesc.h:67
@ OPERAND_GENERIC_5
Definition MCInstrDesc.h:72
Not(const Pred &P) -> Not< Pred >
constexpr bool isSDWA(const T &...O)
Definition SIDefines.h:252
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
@ Offset
Definition DWP.cpp:577
LLVM_ABI void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O)
Create RegSubRegPair from a register MachineOperand.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
constexpr uint64_t maxUIntN(uint64_t N)
Gets the maximum value for a N-bit unsigned integer.
Definition MathExtras.h:208
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI, const MachineInstr &UseMI)
Return false if EXEC is not changed between the def of VReg at DefMI and the use at UseMI.
RegState
Flags to represent properties of register accesses.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ Define
Register definition.
@ NoFlags
No Specific Flags.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
constexpr RegState getKillRegState(bool B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
static const MachineMemOperand::Flags MOCFSB1
Definition SIInstrInfo.h:75
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
Op::Description Desc
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg)
Return the SubReg component from REG_SEQUENCE.
static const MachineMemOperand::Flags MONoClobber
Mark the MMO of a uniform load if there are no potentially clobbering stores on any path from the sta...
Definition SIInstrInfo.h:49
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
MachineInstr * getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, const MachineRegisterInfo &MRI)
Return the defining instruction for a given reg:subreg pair skipping copy like instructions and subre...
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI VirtRegInfo AnalyzeVirtRegInBundle(MachineInstr &MI, Register Reg, SmallVectorImpl< std::pair< MachineInstr *, unsigned > > *Ops=nullptr)
AnalyzeVirtRegInBundle - Analyze how the current instruction or bundle uses a virtual register.
static const MachineMemOperand::Flags MOCooperative
Mark the MMO of cooperative load/store atomics.
Definition SIInstrInfo.h:57
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
@ Xor
Bitwise or logical XOR of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
bool isTargetSpecificOpcode(unsigned Opcode)
Check whether the given Opcode is a target-specific opcode.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned DefaultMemoryClusterDWordsLimit
Definition SIInstrInfo.h:45
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Definition MathExtras.h:249
static const MachineMemOperand::Flags MOLastUse
Mark the MMO of a load as the last use.
Definition SIInstrInfo.h:53
constexpr T reverseBits(T Val)
Reverse the bits in Val.
Definition MathExtras.h:119
static const MachineMemOperand::Flags MOCFSB0
Use two MMO flags to encode the 2-bit cache fill size field for GFX13+.
Definition SIInstrInfo.h:72
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition MathExtras.h:572
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
constexpr RegState getUndefRegState(bool B)
ValueUniformity
Enum describing how values behave with respect to uniformity and divergence, to answer the question: ...
Definition Uniformity.h:18
@ AlwaysUniform
The result value is always uniform.
Definition Uniformity.h:23
@ NeverUniform
The result value can never be assumed to be uniform.
Definition Uniformity.h:26
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
static const MachineMemOperand::Flags MOThreadPrivate
Mark the MMO of accesses to memory locations that are never written to by other threads.
Definition SIInstrInfo.h:68
bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI)
Return false if EXEC is not changed between the def of VReg at DefMI and all its uses.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
Helper struct for the implementation of 3-address conversion to communicate updates made to instructi...
MachineInstr * RemoveMIUse
Other instruction whose def is no longer used by the converted instruction.
static constexpr uint64_t encode(Fields... Values)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr bool all() const
Definition LaneBitmask.h:54
This represents a simple continuous liveness interval for a value.
Summarize the scheduling resources required for an instruction of a particular scheduling class.
Definition MCSchedule.h:124
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Utility to store machine instructions worklist.
Definition SIInstrInfo.h:79
MachineInstr * top() const
Definition SIInstrInfo.h:84
bool isDeferred(MachineInstr *MI)
SetVector< MachineInstr * > & getDeferredList()
void insert(MachineInstr *MI)
A pair composed of a register and a sub-register index.
VirtRegInfo - Information about a virtual register used by a set of operands.
bool Reads
Reads - One of the operands read the virtual register.
bool Writes
Writes - One of the operands writes the virtual register.