LLVM 24.0.0git
AMDGPUInstructionSelector.cpp
Go to the documentation of this file.
1//===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
15#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include <optional>
30
31#define DEBUG_TYPE "amdgpu-isel"
32
33using namespace llvm;
34using namespace MIPatternMatch;
35
36#define GET_GLOBALISEL_IMPL
37#define AMDGPUSubtarget GCNSubtarget
38#include "AMDGPUGenGlobalISel.inc"
39#undef GET_GLOBALISEL_IMPL
40#undef AMDGPUSubtarget
41
43 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
44 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
46#include "AMDGPUGenGlobalISel.inc"
49#include "AMDGPUGenGlobalISel.inc"
51{
52}
53
54const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
55
66
67// Return the wave level SGPR base address if this is a wave address.
69 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
70 ? Def->getOperand(1).getReg()
71 : Register();
72}
73
74bool AMDGPUInstructionSelector::isVCC(Register Reg,
75 const MachineRegisterInfo &MRI) const {
76 // The verifier is oblivious to s1 being a valid value for wavesize registers.
77 if (Reg.isPhysical())
78 return false;
79
80 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
81 const TargetRegisterClass *RC =
83 if (RC) {
84 const LLT Ty = MRI.getType(Reg);
85 if (!Ty.isValid() || Ty.getSizeInBits() != 1)
86 return false;
87 // G_TRUNC s1 result is never vcc.
88 return !mi_match(Reg, MRI, m_GTrunc(m_Reg())) &&
89 RC->hasSuperClassEq(TRI.getBoolRC());
90 }
91
92 const RegisterBank *RB = cast<const RegisterBank *>(RegClassOrBank);
93 return RB->getID() == AMDGPU::VCCRegBankID;
94}
95
96bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
97 unsigned NewOpc) const {
98 MI.setDesc(TII.get(NewOpc));
99 MI.removeOperand(1); // Remove intrinsic ID.
100 MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
101
102 Register DstReg = MI.getOperand(0).getReg();
103 Register SrcReg = MI.getOperand(1).getReg();
104
105 // TODO: This should be legalized to s32 if needed
106 if (MRI->getType(DstReg) == LLT::scalar(1))
107 return false;
108
109 const TargetRegisterClass *DstRC =
110 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
111 const TargetRegisterClass *SrcRC =
112 TRI.getConstrainedRegClassForReg(SrcReg, *MRI);
113 if (!DstRC || DstRC != SrcRC)
114 return false;
115
116 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
117 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
118 return false;
119 const MCInstrDesc &MCID = MI.getDesc();
120 if (MCID.getOperandConstraint(0, MCOI::EARLY_CLOBBER) != -1) {
121 MI.getOperand(0).setIsEarlyClobber(true);
122 }
123 return true;
124}
125
126bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
127 const DebugLoc &DL = I.getDebugLoc();
128 MachineBasicBlock *BB = I.getParent();
129 I.setDesc(TII.get(TargetOpcode::COPY));
130
131 Register DstReg = I.getOperand(0).getReg();
132 Register SrcReg = I.getOperand(1).getReg();
133
134 if (isVCC(DstReg, *MRI)) {
135 if (SrcReg == AMDGPU::SCC) {
136 const TargetRegisterClass *RC =
137 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
138 if (!RC)
139 return true;
140 return RBI.constrainGenericRegister(DstReg, *RC, *MRI);
141 }
142
143 if (!isVCC(SrcReg, *MRI)) {
144 // TODO: Should probably leave the copy and let copyPhysReg expand it.
145 if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI))
146 return false;
147
148 const TargetRegisterClass *SrcRC =
149 TRI.getConstrainedRegClassForReg(SrcReg, *MRI);
150
151 std::optional<ValueAndVReg> ConstVal =
152 getIConstantVRegValWithLookThrough(SrcReg, *MRI, true);
153 if (ConstVal) {
154 unsigned MovOpc =
155 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
156 BuildMI(*BB, &I, DL, TII.get(MovOpc), DstReg)
157 .addImm(ConstVal->Value.getBoolValue() ? -1 : 0);
158 } else {
159 Register MaskedReg = MRI->createVirtualRegister(SrcRC);
160
161 // We can't trust the high bits at this point, so clear them.
162
163 // TODO: Skip masking high bits if def is known boolean.
164
165 if (AMDGPU::getRegBitWidth(SrcRC->getID()) == 16) {
166 assert(Subtarget->useRealTrue16Insts());
167 const int64_t NoMods = 0;
168 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_AND_B16_t16_e64), MaskedReg)
169 .addImm(NoMods)
170 .addImm(1)
171 .addImm(NoMods)
172 .addReg(SrcReg)
173 .addImm(NoMods);
174 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U16_t16_e64), DstReg)
175 .addImm(NoMods)
176 .addImm(0)
177 .addImm(NoMods)
178 .addReg(MaskedReg)
179 .addImm(NoMods);
180 } else {
181 bool IsSGPR = TRI.isSGPRClass(SrcRC);
182 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
183 auto And = BuildMI(*BB, &I, DL, TII.get(AndOpc), MaskedReg)
184 .addImm(1)
185 .addReg(SrcReg);
186 if (IsSGPR)
187 And.setOperandDead(3); // Dead scc
188
189 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg)
190 .addImm(0)
191 .addReg(MaskedReg);
192 }
193 }
194
195 if (!MRI->getRegClassOrNull(SrcReg))
196 MRI->setRegClass(SrcReg, SrcRC);
197 I.eraseFromParent();
198 return true;
199 }
200
201 const TargetRegisterClass *RC =
202 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
203 if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI))
204 return false;
205
206 return true;
207 }
208
209 for (const MachineOperand &MO : I.operands()) {
210 if (MO.getReg().isPhysical())
211 continue;
212
213 const TargetRegisterClass *RC =
214 TRI.getConstrainedRegClassForReg(MO.getReg(), *MRI);
215 if (!RC)
216 continue;
217 RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI);
218 }
219 return true;
220}
221
222bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(MachineInstr &I) const {
223 const DebugLoc &DL = I.getDebugLoc();
224 MachineBasicBlock *BB = I.getParent();
225 Register VCCReg = I.getOperand(1).getReg();
226 MachineInstr *Cmp;
227
228 // Set SCC as a side effect with S_CMP or S_OR.
229 if (STI.hasScalarCompareEq64()) {
230 unsigned CmpOpc =
231 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
232 Cmp = BuildMI(*BB, &I, DL, TII.get(CmpOpc)).addReg(VCCReg).addImm(0);
233 } else {
234 Register DeadDst = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
235 Cmp = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_OR_B64), DeadDst)
236 .addReg(VCCReg)
237 .addReg(VCCReg);
238 }
239
240 constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
241
242 Register DstReg = I.getOperand(0).getReg();
243 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(AMDGPU::SCC);
244
245 I.eraseFromParent();
246 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
247}
248
249bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(MachineInstr &I) const {
250 const DebugLoc &DL = I.getDebugLoc();
251 MachineBasicBlock *BB = I.getParent();
252
253 Register DstReg = I.getOperand(0).getReg();
254 Register SrcReg = I.getOperand(1).getReg();
255 std::optional<ValueAndVReg> Arg =
256 getIConstantVRegValWithLookThrough(I.getOperand(1).getReg(), *MRI);
257
258 if (Arg) {
259 const int64_t Value = Arg->Value.getZExtValue();
260 if (Value == 0) {
261 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
262 BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg).addImm(0);
263 } else {
264 assert(Value == 1);
265 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg).addReg(TRI.getExec());
266 }
267 I.eraseFromParent();
268 return RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI);
269 }
270
271 // RegBankLegalize ensures that SrcReg is bool in reg (high bits are 0).
272 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC).addReg(SrcReg);
273
274 unsigned SelectOpcode =
275 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
276 MachineInstr *Select = BuildMI(*BB, &I, DL, TII.get(SelectOpcode), DstReg)
277 .addReg(TRI.getExec())
278 .addImm(0);
279
280 I.eraseFromParent();
282 return true;
283}
284
285bool AMDGPUInstructionSelector::selectReadAnyLane(MachineInstr &I) const {
286 Register DstReg = I.getOperand(0).getReg();
287 Register SrcReg = I.getOperand(1).getReg();
288
289 const DebugLoc &DL = I.getDebugLoc();
290 MachineBasicBlock *BB = I.getParent();
291
292 auto RFL = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
293 .addReg(SrcReg);
294
295 I.eraseFromParent();
296 constrainSelectedInstRegOperands(*RFL, TII, TRI, RBI);
297 return true;
298}
299
300bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
301 const Register DefReg = I.getOperand(0).getReg();
302 const LLT DefTy = MRI->getType(DefReg);
303
304 // S1 G_PHIs should not be selected in instruction-select, instead:
305 // - divergent S1 G_PHI should go through lane mask merging algorithm
306 // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering
307 // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect
308 if (DefTy == LLT::scalar(1))
309 return false;
310
311 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
312
313 const RegClassOrRegBank &RegClassOrBank =
314 MRI->getRegClassOrRegBank(DefReg);
315
316 const TargetRegisterClass *DefRC =
318 if (!DefRC) {
319 if (!DefTy.isValid()) {
320 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
321 return false;
322 }
323
324 const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
325 DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB);
326 if (!DefRC) {
327 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
328 return false;
329 }
330 }
331
332 // If inputs have register bank, assign corresponding reg class.
333 // Note: registers don't need to have the same reg bank.
334 for (unsigned i = 1; i != I.getNumOperands(); i += 2) {
335 const Register SrcReg = I.getOperand(i).getReg();
336
337 const RegisterBank *RB = MRI->getRegBankOrNull(SrcReg);
338 if (RB) {
339 const LLT SrcTy = MRI->getType(SrcReg);
340 const TargetRegisterClass *SrcRC =
341 TRI.getRegClassForTypeOnBank(SrcTy, *RB);
342 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
343 return false;
344 }
345 }
346
347 I.setDesc(TII.get(TargetOpcode::PHI));
348 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
349}
350
352AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
353 const TargetRegisterClass &SubRC,
354 unsigned SubIdx) const {
355
356 MachineInstr *MI = MO.getParent();
357 MachineBasicBlock *BB = MO.getParent()->getParent();
358 Register DstReg = MRI->createVirtualRegister(&SubRC);
359
360 if (MO.isReg()) {
361 unsigned ComposedSubIdx = TRI.composeSubRegIndices(MO.getSubReg(), SubIdx);
362 Register Reg = MO.getReg();
363 BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
364 .addReg(Reg, {}, ComposedSubIdx);
365
366 return MachineOperand::CreateReg(DstReg, MO.isDef(), MO.isImplicit(),
367 MO.isKill(), MO.isDead(), MO.isUndef(),
368 MO.isEarlyClobber(), 0, MO.isDebug(),
369 MO.isInternalRead());
370 }
371
372 assert(MO.isImm());
373
374 APInt Imm(64, MO.getImm());
375
376 switch (SubIdx) {
377 default:
378 llvm_unreachable("do not know to split immediate with this sub index.");
379 case AMDGPU::sub0:
380 return MachineOperand::CreateImm(Imm.getLoBits(32).getSExtValue());
381 case AMDGPU::sub1:
382 return MachineOperand::CreateImm(Imm.getHiBits(32).getSExtValue());
383 }
384}
385
386static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
387 switch (Opc) {
388 case AMDGPU::G_AND:
389 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
390 case AMDGPU::G_OR:
391 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
392 case AMDGPU::G_XOR:
393 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
394 default:
395 llvm_unreachable("not a bit op");
396 }
397}
398
399bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
400 Register DstReg = I.getOperand(0).getReg();
401 uint64_t Size = TRI.getRegSizeInBits(DstReg, *MRI);
402
403 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
404 if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
405 DstRB->getID() != AMDGPU::VCCRegBankID)
406 return false;
407
408 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
409 STI.isWave64());
410 I.setDesc(TII.get(getLogicalBitOpcode(I.getOpcode(), Is64)));
411
412 // Dead implicit-def of scc
413 I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef
414 true, // isImp
415 false, // isKill
416 true)); // isDead
417 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
418 return true;
419}
420
421bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
422 MachineBasicBlock *BB = I.getParent();
424 Register DstReg = I.getOperand(0).getReg();
425 const DebugLoc &DL = I.getDebugLoc();
426 LLT Ty = MRI->getType(DstReg);
427 if (Ty.isVector())
428 return false;
429
431 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
432 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
433 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
434
435 if (Size == 32) {
436 if (IsSALU) {
437 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
438 MachineInstr *Add =
439 BuildMI(*BB, &I, DL, TII.get(Opc), DstReg)
440 .add(I.getOperand(1))
441 .add(I.getOperand(2))
442 .setOperandDead(3); // Dead scc
443 I.eraseFromParent();
444 constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
445 return true;
446 }
447
448 if (STI.hasAddNoCarryInsts()) {
449 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
450 I.setDesc(TII.get(Opc));
451 I.addOperand(*MF, MachineOperand::CreateImm(0));
452 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
453 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
454 return true;
455 }
456
457 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
458
459 Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass());
460 MachineInstr *Add
461 = BuildMI(*BB, &I, DL, TII.get(Opc), DstReg)
462 .addDef(UnusedCarry, RegState::Dead)
463 .add(I.getOperand(1))
464 .add(I.getOperand(2))
465 .addImm(0);
466 I.eraseFromParent();
467 constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
468 return true;
469 }
470
471 assert(!Sub && "illegal sub should not reach here");
472
473 const TargetRegisterClass &RC
474 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
475 const TargetRegisterClass &HalfRC
476 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
477
478 MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0));
479 MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0));
480 MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1));
481 MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1));
482
483 Register DstLo = MRI->createVirtualRegister(&HalfRC);
484 Register DstHi = MRI->createVirtualRegister(&HalfRC);
485
486 if (IsSALU) {
487 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo)
488 .add(Lo1)
489 .add(Lo2);
490 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi)
491 .add(Hi1)
492 .add(Hi2)
493 .setOperandDead(3); // Dead scc
494 } else {
495 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
496 Register CarryReg = MRI->createVirtualRegister(CarryRC);
497 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_CO_U32_e64), DstLo)
498 .addDef(CarryReg)
499 .add(Lo1)
500 .add(Lo2)
501 .addImm(0);
502 MachineInstr *Addc =
503 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi)
504 .addDef(MRI->createVirtualRegister(CarryRC), RegState::Dead)
505 .add(Hi1)
506 .add(Hi2)
507 .addReg(CarryReg)
508 .addImm(0);
509
510 constrainSelectedInstRegOperands(*Addc, TII, TRI, RBI);
511 }
512
513 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
514 .addReg(DstLo)
515 .addImm(AMDGPU::sub0)
516 .addReg(DstHi)
517 .addImm(AMDGPU::sub1);
518
519
520 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI))
521 return false;
522
523 I.eraseFromParent();
524 return true;
525}
526
527bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
528 MachineInstr &I) const {
529 MachineBasicBlock *BB = I.getParent();
531 const DebugLoc &DL = I.getDebugLoc();
532 Register Dst0Reg = I.getOperand(0).getReg();
533 Register Dst1Reg = I.getOperand(1).getReg();
534 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO ||
535 I.getOpcode() == AMDGPU::G_UADDE;
536 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE ||
537 I.getOpcode() == AMDGPU::G_USUBE;
538
539 if (isVCC(Dst1Reg, *MRI)) {
540 unsigned NoCarryOpc =
541 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
542 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
543 I.setDesc(TII.get(HasCarryIn ? CarryOpc : NoCarryOpc));
544 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
545 I.addOperand(*MF, MachineOperand::CreateImm(0));
546 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
547 return true;
548 }
549
550 Register Src0Reg = I.getOperand(2).getReg();
551 Register Src1Reg = I.getOperand(3).getReg();
552
553 if (HasCarryIn) {
554 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
555 .addReg(I.getOperand(4).getReg());
556 }
557
558 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
559 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
560
561 auto CarryInst = BuildMI(*BB, &I, DL, TII.get(HasCarryIn ? CarryOpc : NoCarryOpc), Dst0Reg)
562 .add(I.getOperand(2))
563 .add(I.getOperand(3));
564
565 if (MRI->use_nodbg_empty(Dst1Reg)) {
566 CarryInst.setOperandDead(3); // Dead scc
567 } else {
568 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg)
569 .addReg(AMDGPU::SCC);
570 if (!MRI->getRegClassOrNull(Dst1Reg))
571 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass);
572 }
573
574 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) ||
575 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) ||
576 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI))
577 return false;
578
579 if (HasCarryIn &&
580 !RBI.constrainGenericRegister(I.getOperand(4).getReg(),
581 AMDGPU::SReg_32RegClass, *MRI))
582 return false;
583
584 I.eraseFromParent();
585 return true;
586}
587
588bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
589 MachineInstr &I) const {
590 MachineBasicBlock *BB = I.getParent();
592 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
593 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
594 MRI->use_nodbg_empty(I.getOperand(1).getReg());
595
596 unsigned Opc;
597 if (Subtarget->hasMADIntraFwdBug())
598 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
599 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
600 else if (UseNoCarry)
601 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
602 : AMDGPU::V_MAD_NC_I64_I32_e64;
603 else
604 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
605
606 if (UseNoCarry)
607 I.removeOperand(1);
608
609 I.setDesc(TII.get(Opc));
610 I.addOperand(*MF, MachineOperand::CreateImm(0));
611 I.addImplicitDefUseOperands(*MF);
612 I.getOperand(0).setIsEarlyClobber(true);
613 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
614 return true;
615}
616
617// TODO: We should probably legalize these to only using 32-bit results.
618bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
619 MachineBasicBlock *BB = I.getParent();
620 Register DstReg = I.getOperand(0).getReg();
621 Register SrcReg = I.getOperand(1).getReg();
622 LLT DstTy = MRI->getType(DstReg);
623 LLT SrcTy = MRI->getType(SrcReg);
624 const uint64_t SrcSize = SrcTy.getSizeInBits();
625 uint64_t DstSize = DstTy.getSizeInBits();
626
627 // TODO: Should handle any multiple of 32 offset.
628 int64_t Offset = I.getOperand(2).getImm();
629 if (Offset % 32 != 0 || DstSize > 128)
630 return false;
631
632 // 16-bit operations really use 32-bit registers.
633 // FIXME: Probably should not allow 16-bit G_EXTRACT results.
634 if (DstSize == 16)
635 DstSize = 32;
636
637 const TargetRegisterClass *DstRC =
638 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
639 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
640 return false;
641
642 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI);
643 const TargetRegisterClass *SrcRC =
644 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank);
645 if (!SrcRC)
646 return false;
647 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Offset / 32,
648 DstSize / 32);
649 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
650 if (!SrcRC)
651 return false;
652
653 SrcReg = constrainOperandRegClass(*MF, TRI, *MRI, TII, RBI, I,
654 *SrcRC, I.getOperand(1));
655 const DebugLoc &DL = I.getDebugLoc();
656 BuildMI(*BB, &I, DL, TII.get(TargetOpcode::COPY), DstReg)
657 .addReg(SrcReg, {}, SubReg);
658
659 I.eraseFromParent();
660 return true;
661}
662
663bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
664 Register Dst = MI.getOperand(0).getReg();
665 Register Src0 = MI.getOperand(1).getReg();
666 Register Src1 = MI.getOperand(2).getReg();
667
668 LLT Src0Ty = MRI->getType(Src0);
669 LLT Src1Ty = MRI->getType(Src1);
670
671 const RegisterBank *DstBank = RBI.getRegBank(Dst, *MRI, TRI);
672 const RegisterBank *Src0Bank = RBI.getRegBank(Src0, *MRI, TRI);
673 const RegisterBank *Src1Bank = RBI.getRegBank(Src1, *MRI, TRI);
674 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
675
676 Register ShiftSrc0;
677 Register ShiftSrc1;
678
679 const DebugLoc &DL = MI.getDebugLoc();
680 MachineBasicBlock *BB = MI.getParent();
681
682 // VGPR case
683 if (IsVector) {
684 // If source are both VGPR16, use REG_SEQUENCE with lo16/hi16 subregisters
685 if (Src0Bank->getID() == AMDGPU::VGPRRegBankID &&
686 Src1Bank->getID() == AMDGPU::VGPRRegBankID &&
687 Src0Ty == LLT::scalar(16) && Src1Ty == LLT::scalar(16)) {
688 BuildMI(*BB, MI, DL, TII.get(TargetOpcode::REG_SEQUENCE), Dst)
689 .addReg(Src0)
690 .addImm(AMDGPU::lo16)
691 .addReg(Src1)
692 .addImm(AMDGPU::hi16);
693
694 if (!RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI))
695 return false;
696
697 MI.eraseFromParent();
698 return true;
699 }
700
701 // Otherwise, use V_LSHL_OR_B32_e64
702 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
703 auto MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_AND_B32_e32), TmpReg)
704 .addImm(0xFFFF)
705 .addReg(Src0);
706 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
707
708 MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_LSHL_OR_B32_e64), Dst)
709 .addReg(Src1)
710 .addImm(16)
711 .addReg(TmpReg);
712 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
713
714 MI.eraseFromParent();
715 return true;
716 }
717
718 // SGPR case -> S_PACK_*_B32_B16
719 // With multiple uses of the shift, this will duplicate the shift and
720 // increase register pressure.
721 //
722 // (merge (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
723 // => (S_PACK_HH_B32_B16 $src0, $src1)
724 // (merge (lshr_oneuse SReg_32:$src0, 16), $src1)
725 // => (S_PACK_HL_B32_B16 $src0, $src1)
726 // (merge $src0, (lshr_oneuse SReg_32:$src1, 16))
727 // => (S_PACK_LH_B32_B16 $src0, $src1)
728 // (merge $src0, $src1)
729 // => (S_PACK_LL_B32_B16 $src0, $src1)
730
731 bool Shift0 = mi_match(
732 Src0, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc0), m_SpecificICst(16))));
733
734 bool Shift1 = mi_match(
735 Src1, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc1), m_SpecificICst(16))));
736
737 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
738 if (Shift0 && Shift1) {
739 Opc = AMDGPU::S_PACK_HH_B32_B16;
740 MI.getOperand(1).setReg(ShiftSrc0);
741 MI.getOperand(2).setReg(ShiftSrc1);
742 } else if (Shift1) {
743 Opc = AMDGPU::S_PACK_LH_B32_B16;
744 MI.getOperand(2).setReg(ShiftSrc1);
745 } else if (Shift0) {
746 auto ConstSrc1 =
747 getAnyConstantVRegValWithLookThrough(Src1, *MRI, true, true);
748 if (ConstSrc1 && ConstSrc1->Value == 0) {
749 // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
750 auto MIB = BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst)
751 .addReg(ShiftSrc0)
752 .addImm(16)
753 .setOperandDead(3); // Dead scc
754
755 MI.eraseFromParent();
756 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
757 return true;
758 }
759 if (STI.hasSPackHL()) {
760 Opc = AMDGPU::S_PACK_HL_B32_B16;
761 MI.getOperand(1).setReg(ShiftSrc0);
762 }
763 }
764
765 MI.setDesc(TII.get(Opc));
766 constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
767 return true;
768}
769
770// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
771// s32 pieces into the destination with a REG_SEQUENCE.
772bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
773 MachineBasicBlock *BB = MI.getParent();
774 const DebugLoc &DL = MI.getDebugLoc();
775 Register DstReg = MI.getOperand(0).getReg();
776 const uint64_t DstSize = MRI->getType(DstReg).getSizeInBits();
777 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
778 const unsigned NumSrc = MI.getNumOperands() - 1;
779
780 // Pack each pair of s16 sources into an s32.
782 for (unsigned I = 0; I != NumSrc; I += 2) {
783 Register S32 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
784 auto Pack = BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_PACK_LL_B32_B16), S32)
785 .addReg(MI.getOperand(I + 1).getReg())
786 .addReg(MI.getOperand(I + 2).getReg());
787 constrainSelectedInstRegOperands(*Pack, TII, TRI, RBI);
788 S32Regs.push_back(S32);
789 }
790
791 // Combine the s32 pieces into the destination with a REG_SEQUENCE.
792 const TargetRegisterClass *DstRC =
793 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
794 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
795 return false;
796 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, /*EltSize=*/4);
797 auto MIB = BuildMI(*BB, MI, DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg);
798 for (size_t I = 0, E = S32Regs.size(); I != E; ++I)
799 MIB.addReg(S32Regs[I]).addImm(SubRegs[I]);
800
801 MI.eraseFromParent();
802 return true;
803}
804
805bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
806 MachineBasicBlock *BB = MI.getParent();
807 Register DstReg = MI.getOperand(0).getReg();
808 LLT DstTy = MRI->getType(DstReg);
809 LLT SrcTy = MRI->getType(MI.getOperand(1).getReg());
810
811 const uint64_t SrcSize = SrcTy.getSizeInBits();
812 if (SrcSize < 32) {
813 // Handle s32 <- G_MERGE_VALUES s16, s16
814 if (SrcSize == 16 && DstTy.getSizeInBits() == 32 &&
815 MI.getNumOperands() == 3) {
816 return selectS16MergeToS32(MI);
817 }
818 // With true16 a scalar s16 is a register type, so a scalar wider than 32
819 // bits can be built from s16 pieces.
820 bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
821 DstTy.getSizeInBits() % 32 == 0;
822
823 // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
824 if (IsWideS16Merge &&
825 RBI.getRegBank(DstReg, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
826 return selectS16MergeToWide(MI);
827
828 // A VGPR wide s16 merge falls through to the generic path below.
829 if (!IsWideS16Merge)
830 return selectImpl(MI, *CoverageInfo);
831 }
832
833 const DebugLoc &DL = MI.getDebugLoc();
834 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
835 const uint64_t DstSize = DstTy.getSizeInBits();
836 const TargetRegisterClass *DstRC =
837 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
838 if (!DstRC)
839 return false;
840
841 unsigned SrcSizeEnc = static_cast<unsigned>(SrcSize);
842 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, SrcSizeEnc / 8);
843 MachineInstrBuilder MIB =
844 BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg);
845 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
846 MachineOperand &Src = MI.getOperand(I + 1);
847 Register SrcReg = Src.getReg();
848 MIB.addReg(SrcReg, getUndefRegState(Src.isUndef()));
849 MIB.addImm(SubRegs[I]);
850
851 const TargetRegisterClass *SrcRC =
852 TRI.getConstrainedRegClassForReg(SrcReg, *MRI);
853 if (SrcRC && !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
854 return false;
855 }
856
857 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
858 return false;
859
860 MI.eraseFromParent();
861 return true;
862}
863
864bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
865 MachineBasicBlock *BB = MI.getParent();
866 const int NumDst = MI.getNumOperands() - 1;
867
868 MachineOperand &Src = MI.getOperand(NumDst);
869
870 Register SrcReg = Src.getReg();
871 Register DstReg0 = MI.getOperand(0).getReg();
872 LLT DstTy = MRI->getType(DstReg0);
873 LLT SrcTy = MRI->getType(SrcReg);
874
875 const uint64_t DstSize = DstTy.getSizeInBits();
876 const uint64_t SrcSize = SrcTy.getSizeInBits();
877 const DebugLoc &DL = MI.getDebugLoc();
878 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI);
879
880 const TargetRegisterClass *SrcRC =
881 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank);
882
883 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
884 // source, and this relies on the fact that the same subregister indices are
885 // used for both.
886 unsigned DstSizeEnc = static_cast<unsigned>(DstSize);
887 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSizeEnc / 8);
888 for (int I = 0, E = NumDst; I != E; ++I) {
889 Register DstReg = MI.getOperand(I).getReg();
890 // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
891 if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
892 SubRegs[I] == AMDGPU::hi16) {
893 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg)
894 .addReg(SrcReg)
895 .addImm(16);
896 } else {
897 BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), DstReg)
898 .addReg(SrcReg, {}, SubRegs[I]);
899
900 // Make sure the subregister index is valid for the source register.
901 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
902 }
903
904 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
905 return false;
906
907 const TargetRegisterClass *DstRC =
908 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
909 if (DstRC && !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
910 return false;
911 }
912
913 MI.eraseFromParent();
914 return true;
915}
916
917bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
918 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
919 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
920
921 Register Src0 = MI.getOperand(1).getReg();
922 Register Src1 = MI.getOperand(2).getReg();
923 LLT SrcTy = MRI->getType(Src0);
924 const uint64_t SrcSize = SrcTy.getSizeInBits();
925
926 // BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
927 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
928 return selectG_MERGE_VALUES(MI);
929 }
930
931 // Selection logic below is for V2S16 only.
932 // For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
933 Register Dst = MI.getOperand(0).getReg();
934 if (MRI->getType(Dst) != LLT::fixed_vector(2, 16) ||
935 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
936 SrcTy != LLT::scalar(32)))
937 return selectImpl(MI, *CoverageInfo);
938
939 const RegisterBank *DstBank = RBI.getRegBank(Dst, *MRI, TRI);
940 if (DstBank->getID() == AMDGPU::AGPRRegBankID)
941 return false;
942
943 assert(DstBank->getID() == AMDGPU::SGPRRegBankID ||
944 DstBank->getID() == AMDGPU::VGPRRegBankID);
945 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
946
947 const DebugLoc &DL = MI.getDebugLoc();
948 MachineBasicBlock *BB = MI.getParent();
949
950 // First, before trying TableGen patterns, check if both sources are
951 // constants. In those cases, we can trivially compute the final constant
952 // and emit a simple move.
953 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(Src1, *MRI, true, true);
954 if (ConstSrc1) {
955 auto ConstSrc0 =
956 getAnyConstantVRegValWithLookThrough(Src0, *MRI, true, true);
957 if (ConstSrc0) {
958 const int64_t K0 = ConstSrc0->Value.getSExtValue();
959 const int64_t K1 = ConstSrc1->Value.getSExtValue();
960 uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
961 uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
962 uint32_t Imm = Lo16 | (Hi16 << 16);
963
964 // VALU
965 if (IsVector) {
966 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), Dst).addImm(Imm);
967 MI.eraseFromParent();
968 return RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI);
969 }
970
971 // SALU
972 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), Dst).addImm(Imm);
973 MI.eraseFromParent();
974 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI);
975 }
976 }
977
978 // Now try TableGen patterns.
979 if (selectImpl(MI, *CoverageInfo))
980 return true;
981
982 // TODO: This should probably be a combine somewhere
983 // (build_vector $src0, undef) -> copy $src0
984 MachineInstr *Src1Def = getDefIgnoringCopies(Src1, *MRI);
985 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
986 MI.setDesc(TII.get(AMDGPU::COPY));
987 MI.removeOperand(2);
988 const auto &RC =
989 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
990 return RBI.constrainGenericRegister(Dst, RC, *MRI) &&
991 RBI.constrainGenericRegister(Src0, RC, *MRI);
992 }
993
994 return selectS16MergeToS32(MI);
995}
996
997bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
998 const MachineOperand &MO = I.getOperand(0);
999
1000 // FIXME: Interface for getConstrainedRegClassForReg needs work. The
1001 // regbank check here is to know why getConstrainedRegClassForReg failed.
1002 const TargetRegisterClass *RC =
1003 TRI.getConstrainedRegClassForReg(MO.getReg(), *MRI);
1004 if ((!RC && !MRI->getRegBankOrNull(MO.getReg())) ||
1005 (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI))) {
1006 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
1007 return true;
1008 }
1009
1010 return false;
1011}
1012
1013bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
1014 MachineBasicBlock *BB = I.getParent();
1015
1016 Register DstReg = I.getOperand(0).getReg();
1017 Register Src0Reg = I.getOperand(1).getReg();
1018 Register Src1Reg = I.getOperand(2).getReg();
1019 LLT Src1Ty = MRI->getType(Src1Reg);
1020
1021 uint64_t DstSize = MRI->getType(DstReg).getSizeInBits();
1022 uint64_t InsSize = Src1Ty.getSizeInBits();
1023
1024 int64_t Offset = I.getOperand(3).getImm();
1025
1026 // FIXME: These cases should have been illegal and unnecessary to check here.
1027 if (Offset % 32 != 0 || InsSize % 32 != 0)
1028 return false;
1029
1030 // Currently not handled by getSubRegFromChannel.
1031 if (InsSize > 128)
1032 return false;
1033
1034 unsigned SubReg = TRI.getSubRegFromChannel(Offset / 32, InsSize / 32);
1035 if (SubReg == AMDGPU::NoSubRegister)
1036 return false;
1037
1038 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
1039 const TargetRegisterClass *DstRC =
1040 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
1041 if (!DstRC)
1042 return false;
1043
1044 const RegisterBank *Src0Bank = RBI.getRegBank(Src0Reg, *MRI, TRI);
1045 const RegisterBank *Src1Bank = RBI.getRegBank(Src1Reg, *MRI, TRI);
1046 const TargetRegisterClass *Src0RC =
1047 TRI.getRegClassForSizeOnBank(DstSize, *Src0Bank);
1048 const TargetRegisterClass *Src1RC =
1049 TRI.getRegClassForSizeOnBank(InsSize, *Src1Bank);
1050
1051 // Deal with weird cases where the class only partially supports the subreg
1052 // index.
1053 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1054 if (!Src0RC || !Src1RC)
1055 return false;
1056
1057 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
1058 !RBI.constrainGenericRegister(Src0Reg, *Src0RC, *MRI) ||
1059 !RBI.constrainGenericRegister(Src1Reg, *Src1RC, *MRI))
1060 return false;
1061
1062 const DebugLoc &DL = I.getDebugLoc();
1063 BuildMI(*BB, &I, DL, TII.get(TargetOpcode::INSERT_SUBREG), DstReg)
1064 .addReg(Src0Reg)
1065 .addReg(Src1Reg)
1066 .addImm(SubReg);
1067
1068 I.eraseFromParent();
1069 return true;
1070}
1071
1072bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
1073 Register DstReg = MI.getOperand(0).getReg();
1074 Register SrcReg = MI.getOperand(1).getReg();
1075 Register OffsetReg = MI.getOperand(2).getReg();
1076 Register WidthReg = MI.getOperand(3).getReg();
1077
1078 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1079 "scalar BFX instructions are expanded in regbankselect");
1080 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1081 "64-bit vector BFX instructions are expanded in regbankselect");
1082
1083 const DebugLoc &DL = MI.getDebugLoc();
1084 MachineBasicBlock *MBB = MI.getParent();
1085
1086 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
1087 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1088 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), DstReg)
1089 .addReg(SrcReg)
1090 .addReg(OffsetReg)
1091 .addReg(WidthReg);
1092 MI.eraseFromParent();
1093 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1094 return true;
1095}
1096
1097bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
1098 if (STI.getLDSBankCount() != 16)
1099 return selectImpl(MI, *CoverageInfo);
1100
1101 Register Dst = MI.getOperand(0).getReg();
1102 Register Src0 = MI.getOperand(2).getReg();
1103 Register M0Val = MI.getOperand(6).getReg();
1104 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) ||
1105 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) ||
1106 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI))
1107 return false;
1108
1109 // This requires 2 instructions. It is possible to write a pattern to support
1110 // this, but the generated isel emitter doesn't correctly deal with multiple
1111 // output instructions using the same physical register input. The copy to m0
1112 // is incorrectly placed before the second instruction.
1113 //
1114 // TODO: Match source modifiers.
1115
1116 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1117 const DebugLoc &DL = MI.getDebugLoc();
1118 MachineBasicBlock *MBB = MI.getParent();
1119
1120 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1121 .addReg(M0Val);
1122 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov)
1123 .addImm(2)
1124 .addImm(MI.getOperand(4).getImm()) // $attr
1125 .addImm(MI.getOperand(3).getImm()); // $attrchan
1126
1127 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_P1LV_F16), Dst)
1128 .addImm(0) // $src0_modifiers
1129 .addReg(Src0) // $src0
1130 .addImm(MI.getOperand(4).getImm()) // $attr
1131 .addImm(MI.getOperand(3).getImm()) // $attrchan
1132 .addImm(0) // $src2_modifiers
1133 .addReg(InterpMov) // $src2 - 2 f16 values selected by high
1134 .addImm(MI.getOperand(5).getImm()) // $high
1135 .addImm(0) // $clamp
1136 .addImm(0); // $omod
1137
1138 MI.eraseFromParent();
1139 return true;
1140}
1141
1142// Writelane is special in that it can use SGPR and M0 (which would normally
1143// count as using the constant bus twice - but in this case it is allowed since
1144// the lane selector doesn't count as a use of the constant bus). However, it is
1145// still required to abide by the 1 SGPR rule. Fix this up if we might have
1146// multiple SGPRs.
1147bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
1148 // With a constant bus limit of at least 2, there's no issue.
1149 if (STI.getConstantBusLimit(AMDGPU::V_WRITELANE_B32) > 1)
1150 return selectImpl(MI, *CoverageInfo);
1151
1152 MachineBasicBlock *MBB = MI.getParent();
1153 const DebugLoc &DL = MI.getDebugLoc();
1154 Register VDst = MI.getOperand(0).getReg();
1155 Register Val = MI.getOperand(2).getReg();
1156 Register LaneSelect = MI.getOperand(3).getReg();
1157 Register VDstIn = MI.getOperand(4).getReg();
1158
1159 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_WRITELANE_B32), VDst);
1160
1161 std::optional<ValueAndVReg> ConstSelect =
1162 getIConstantVRegValWithLookThrough(LaneSelect, *MRI);
1163 if (ConstSelect) {
1164 // The selector has to be an inline immediate, so we can use whatever for
1165 // the other operands.
1166 MIB.addReg(Val);
1167 MIB.addImm(ConstSelect->Value.getSExtValue() &
1168 maskTrailingOnes<uint64_t>(STI.getWavefrontSizeLog2()));
1169 } else {
1170 std::optional<ValueAndVReg> ConstVal =
1172
1173 // If the value written is an inline immediate, we can get away without a
1174 // copy to m0.
1175 if (ConstVal && AMDGPU::isInlinableLiteral32(
1176 static_cast<int32_t>(ConstVal->Value.getSExtValue()),
1177 STI.hasInv2PiInlineImm())) {
1178 MIB.addImm(ConstVal->Value.getSExtValue());
1179 MIB.addReg(LaneSelect);
1180 } else {
1181 MIB.addReg(Val);
1182
1183 // If the lane selector was originally in a VGPR and copied with
1184 // readfirstlane, there's a hazard to read the same SGPR from the
1185 // VALU. Constrain to a different SGPR to help avoid needing a nop later.
1186 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI);
1187
1188 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1189 .addReg(LaneSelect);
1190 MIB.addReg(AMDGPU::M0);
1191 }
1192 }
1193
1194 MIB.addReg(VDstIn);
1195
1196 MI.eraseFromParent();
1197 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1198 return true;
1199}
1200
1201// We need to handle this here because tablegen doesn't support matching
1202// instructions with multiple outputs.
1203bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
1204 Register Dst0 = MI.getOperand(0).getReg();
1205 Register Dst1 = MI.getOperand(1).getReg();
1206
1207 LLT Ty = MRI->getType(Dst0);
1208 unsigned Opc;
1209 if (Ty == LLT::scalar(32))
1210 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1211 else if (Ty == LLT::scalar(64))
1212 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1213 else
1214 return false;
1215
1216 // TODO: Match source modifiers.
1217
1218 const DebugLoc &DL = MI.getDebugLoc();
1219 MachineBasicBlock *MBB = MI.getParent();
1220
1221 Register Numer = MI.getOperand(3).getReg();
1222 Register Denom = MI.getOperand(4).getReg();
1223 unsigned ChooseDenom = static_cast<unsigned>(MI.getOperand(5).getImm());
1224
1225 Register Src0 = ChooseDenom != 0 ? Numer : Denom;
1226
1227 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), Dst0)
1228 .addDef(Dst1)
1229 .addImm(0) // $src0_modifiers
1230 .addUse(Src0) // $src0
1231 .addImm(0) // $src1_modifiers
1232 .addUse(Denom) // $src1
1233 .addImm(0) // $src2_modifiers
1234 .addUse(Numer) // $src2
1235 .addImm(0) // $clamp
1236 .addImm(0); // $omod
1237
1238 MI.eraseFromParent();
1239 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1240 return true;
1241}
1242
1243bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1244 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(I).getIntrinsicID();
1245 switch (IntrinsicID) {
1246 case Intrinsic::amdgcn_if_break: {
1247 MachineBasicBlock *BB = I.getParent();
1248
1249 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1250 // SelectionDAG uses for wave32 vs wave64.
1251 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK))
1252 .add(I.getOperand(0))
1253 .add(I.getOperand(2))
1254 .add(I.getOperand(3))
1255 .setOperandDead(3); // implicit-def $scc
1256
1257 Register DstReg = I.getOperand(0).getReg();
1258 Register Src0Reg = I.getOperand(2).getReg();
1259 Register Src1Reg = I.getOperand(3).getReg();
1260
1261 I.eraseFromParent();
1262
1263 for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1264 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass());
1265
1266 return true;
1267 }
1268 case Intrinsic::amdgcn_interp_p1_f16:
1269 return selectInterpP1F16(I);
1270 case Intrinsic::amdgcn_wqm:
1271 return constrainCopyLikeIntrin(I, AMDGPU::WQM);
1272 case Intrinsic::amdgcn_softwqm:
1273 return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM);
1274 case Intrinsic::amdgcn_strict_wwm:
1275 case Intrinsic::amdgcn_wwm:
1276 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WWM);
1277 case Intrinsic::amdgcn_strict_wqm:
1278 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WQM);
1279 case Intrinsic::amdgcn_writelane:
1280 return selectWritelane(I);
1281 case Intrinsic::amdgcn_div_scale:
1282 return selectDivScale(I);
1283 case Intrinsic::amdgcn_ballot:
1284 return selectBallot(I);
1285 case Intrinsic::amdgcn_reloc_constant:
1286 return selectRelocConstant(I);
1287 case Intrinsic::amdgcn_groupstaticsize:
1288 return selectGroupStaticSize(I);
1289 case Intrinsic::returnaddress:
1290 return selectReturnAddress(I);
1291 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1292 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1293 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1294 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1295 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1296 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1297 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1298 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1299 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1300 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1301 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1302 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1303 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1304 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1305 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1306 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1307 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1308 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1309 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1310 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1311 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1312 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1313 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1314 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1315 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1316 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1317 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1318 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1319 return selectSMFMACIntrin(I);
1320 case Intrinsic::amdgcn_permlane16_swap:
1321 case Intrinsic::amdgcn_permlane32_swap:
1322 return selectPermlaneSwapIntrin(I, IntrinsicID);
1323 case Intrinsic::amdgcn_wave_shuffle:
1324 return selectWaveShuffleIntrin(I);
1325 default:
1326 return selectImpl(I, *CoverageInfo);
1327 }
1328}
1329
1331 const GCNSubtarget &ST) {
1332 if (Size != 16 && Size != 32 && Size != 64)
1333 return -1;
1334
1335 if (Size == 16 && !ST.has16BitInsts())
1336 return -1;
1337
1338 const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc,
1339 unsigned FakeS16Opc, unsigned S32Opc,
1340 unsigned S64Opc) {
1341 if (Size == 16)
1342 return ST.hasTrue16BitInsts()
1343 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1344 : S16Opc;
1345 if (Size == 32)
1346 return S32Opc;
1347 return S64Opc;
1348 };
1349
1350 switch (P) {
1351 default:
1352 llvm_unreachable("Unknown condition code!");
1353 case CmpInst::ICMP_NE:
1354 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1355 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1356 AMDGPU::V_CMP_NE_U64_e64);
1357 case CmpInst::ICMP_EQ:
1358 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1359 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1360 AMDGPU::V_CMP_EQ_U64_e64);
1361 case CmpInst::ICMP_SGT:
1362 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1363 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1364 AMDGPU::V_CMP_GT_I64_e64);
1365 case CmpInst::ICMP_SGE:
1366 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1367 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1368 AMDGPU::V_CMP_GE_I64_e64);
1369 case CmpInst::ICMP_SLT:
1370 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1371 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1372 AMDGPU::V_CMP_LT_I64_e64);
1373 case CmpInst::ICMP_SLE:
1374 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1375 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1376 AMDGPU::V_CMP_LE_I64_e64);
1377 case CmpInst::ICMP_UGT:
1378 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1379 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1380 AMDGPU::V_CMP_GT_U64_e64);
1381 case CmpInst::ICMP_UGE:
1382 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1383 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1384 AMDGPU::V_CMP_GE_U64_e64);
1385 case CmpInst::ICMP_ULT:
1386 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1387 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1388 AMDGPU::V_CMP_LT_U64_e64);
1389 case CmpInst::ICMP_ULE:
1390 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1391 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1392 AMDGPU::V_CMP_LE_U64_e64);
1393
1394 case CmpInst::FCMP_OEQ:
1395 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1396 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1397 AMDGPU::V_CMP_EQ_F64_e64);
1398 case CmpInst::FCMP_OGT:
1399 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1400 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1401 AMDGPU::V_CMP_GT_F64_e64);
1402 case CmpInst::FCMP_OGE:
1403 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1404 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1405 AMDGPU::V_CMP_GE_F64_e64);
1406 case CmpInst::FCMP_OLT:
1407 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1408 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1409 AMDGPU::V_CMP_LT_F64_e64);
1410 case CmpInst::FCMP_OLE:
1411 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1412 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1413 AMDGPU::V_CMP_LE_F64_e64);
1414 case CmpInst::FCMP_ONE:
1415 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1416 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1417 AMDGPU::V_CMP_NEQ_F64_e64);
1418 case CmpInst::FCMP_ORD:
1419 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1420 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1421 AMDGPU::V_CMP_O_F64_e64);
1422 case CmpInst::FCMP_UNO:
1423 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1424 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1425 AMDGPU::V_CMP_U_F64_e64);
1426 case CmpInst::FCMP_UEQ:
1427 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1428 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1429 AMDGPU::V_CMP_NLG_F64_e64);
1430 case CmpInst::FCMP_UGT:
1431 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1432 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1433 AMDGPU::V_CMP_NLE_F64_e64);
1434 case CmpInst::FCMP_UGE:
1435 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1436 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1437 AMDGPU::V_CMP_NLT_F64_e64);
1438 case CmpInst::FCMP_ULT:
1439 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1440 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1441 AMDGPU::V_CMP_NGE_F64_e64);
1442 case CmpInst::FCMP_ULE:
1443 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1444 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1445 AMDGPU::V_CMP_NGT_F64_e64);
1446 case CmpInst::FCMP_UNE:
1447 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1448 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1449 AMDGPU::V_CMP_NEQ_F64_e64);
1450 case CmpInst::FCMP_TRUE:
1451 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1452 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1453 AMDGPU::V_CMP_TRU_F64_e64);
1455 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1456 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1457 AMDGPU::V_CMP_F_F64_e64);
1458 }
1459}
1460
1461int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1462 uint64_t Size) const {
1463 if (Size == 64) {
1464 if (!STI.hasScalarCompareEq64())
1465 return -1;
1466
1467 switch (P) {
1468 case CmpInst::ICMP_NE:
1469 return AMDGPU::S_CMP_LG_U64;
1470 case CmpInst::ICMP_EQ:
1471 return AMDGPU::S_CMP_EQ_U64;
1472 default:
1473 return -1;
1474 }
1475 }
1476
1477 if (Size == 32) {
1478 switch (P) {
1479 case CmpInst::ICMP_NE:
1480 return AMDGPU::S_CMP_LG_U32;
1481 case CmpInst::ICMP_EQ:
1482 return AMDGPU::S_CMP_EQ_U32;
1483 case CmpInst::ICMP_SGT:
1484 return AMDGPU::S_CMP_GT_I32;
1485 case CmpInst::ICMP_SGE:
1486 return AMDGPU::S_CMP_GE_I32;
1487 case CmpInst::ICMP_SLT:
1488 return AMDGPU::S_CMP_LT_I32;
1489 case CmpInst::ICMP_SLE:
1490 return AMDGPU::S_CMP_LE_I32;
1491 case CmpInst::ICMP_UGT:
1492 return AMDGPU::S_CMP_GT_U32;
1493 case CmpInst::ICMP_UGE:
1494 return AMDGPU::S_CMP_GE_U32;
1495 case CmpInst::ICMP_ULT:
1496 return AMDGPU::S_CMP_LT_U32;
1497 case CmpInst::ICMP_ULE:
1498 return AMDGPU::S_CMP_LE_U32;
1499 case CmpInst::FCMP_OEQ:
1500 return AMDGPU::S_CMP_EQ_F32;
1501 case CmpInst::FCMP_OGT:
1502 return AMDGPU::S_CMP_GT_F32;
1503 case CmpInst::FCMP_OGE:
1504 return AMDGPU::S_CMP_GE_F32;
1505 case CmpInst::FCMP_OLT:
1506 return AMDGPU::S_CMP_LT_F32;
1507 case CmpInst::FCMP_OLE:
1508 return AMDGPU::S_CMP_LE_F32;
1509 case CmpInst::FCMP_ONE:
1510 return AMDGPU::S_CMP_LG_F32;
1511 case CmpInst::FCMP_ORD:
1512 return AMDGPU::S_CMP_O_F32;
1513 case CmpInst::FCMP_UNO:
1514 return AMDGPU::S_CMP_U_F32;
1515 case CmpInst::FCMP_UEQ:
1516 return AMDGPU::S_CMP_NLG_F32;
1517 case CmpInst::FCMP_UGT:
1518 return AMDGPU::S_CMP_NLE_F32;
1519 case CmpInst::FCMP_UGE:
1520 return AMDGPU::S_CMP_NLT_F32;
1521 case CmpInst::FCMP_ULT:
1522 return AMDGPU::S_CMP_NGE_F32;
1523 case CmpInst::FCMP_ULE:
1524 return AMDGPU::S_CMP_NGT_F32;
1525 case CmpInst::FCMP_UNE:
1526 return AMDGPU::S_CMP_NEQ_F32;
1527 default:
1528 llvm_unreachable("Unknown condition code!");
1529 }
1530 }
1531
1532 if (Size == 16) {
1533 if (!STI.hasSALUFloatInsts())
1534 return -1;
1535
1536 switch (P) {
1537 case CmpInst::FCMP_OEQ:
1538 return AMDGPU::S_CMP_EQ_F16;
1539 case CmpInst::FCMP_OGT:
1540 return AMDGPU::S_CMP_GT_F16;
1541 case CmpInst::FCMP_OGE:
1542 return AMDGPU::S_CMP_GE_F16;
1543 case CmpInst::FCMP_OLT:
1544 return AMDGPU::S_CMP_LT_F16;
1545 case CmpInst::FCMP_OLE:
1546 return AMDGPU::S_CMP_LE_F16;
1547 case CmpInst::FCMP_ONE:
1548 return AMDGPU::S_CMP_LG_F16;
1549 case CmpInst::FCMP_ORD:
1550 return AMDGPU::S_CMP_O_F16;
1551 case CmpInst::FCMP_UNO:
1552 return AMDGPU::S_CMP_U_F16;
1553 case CmpInst::FCMP_UEQ:
1554 return AMDGPU::S_CMP_NLG_F16;
1555 case CmpInst::FCMP_UGT:
1556 return AMDGPU::S_CMP_NLE_F16;
1557 case CmpInst::FCMP_UGE:
1558 return AMDGPU::S_CMP_NLT_F16;
1559 case CmpInst::FCMP_ULT:
1560 return AMDGPU::S_CMP_NGE_F16;
1561 case CmpInst::FCMP_ULE:
1562 return AMDGPU::S_CMP_NGT_F16;
1563 case CmpInst::FCMP_UNE:
1564 return AMDGPU::S_CMP_NEQ_F16;
1565 default:
1566 llvm_unreachable("Unknown condition code!");
1567 }
1568 }
1569
1570 return -1;
1571}
1572
1573bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const {
1574
1575 MachineBasicBlock *BB = I.getParent();
1576 const DebugLoc &DL = I.getDebugLoc();
1577
1578 Register SrcReg = I.getOperand(2).getReg();
1579 uint64_t Size = TRI.getRegSizeInBits(SrcReg, *MRI);
1580
1581 auto Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
1582
1583 Register CCReg = I.getOperand(0).getReg();
1584 if (!isVCC(CCReg, *MRI)) {
1585 int Opcode = getS_CMPOpcode(Pred, Size);
1586 if (Opcode == -1)
1587 return false;
1588 MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode))
1589 .add(I.getOperand(2))
1590 .add(I.getOperand(3));
1591 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg)
1592 .addReg(AMDGPU::SCC);
1593 constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI);
1594 bool Ret =
1595 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI);
1596 I.eraseFromParent();
1597 return Ret;
1598 }
1599
1600 if (I.getOpcode() == AMDGPU::G_FCMP)
1601 return false;
1602
1603 int Opcode = getV_CMPOpcode(Pred, Size, *Subtarget);
1604 if (Opcode == -1)
1605 return false;
1606
1607 MachineInstrBuilder ICmp;
1608 // t16 instructions
1609 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers)) {
1610 ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode), I.getOperand(0).getReg())
1611 .addImm(0)
1612 .add(I.getOperand(2))
1613 .addImm(0)
1614 .add(I.getOperand(3))
1615 .addImm(0); // op_sel
1616 } else {
1617 ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode), I.getOperand(0).getReg())
1618 .add(I.getOperand(2))
1619 .add(I.getOperand(3));
1620 }
1621
1622 RBI.constrainGenericRegister(ICmp->getOperand(0).getReg(),
1623 *TRI.getBoolRC(), *MRI);
1624 constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI);
1625 I.eraseFromParent();
1626 return true;
1627}
1628
1629// Ballot has to zero bits in input lane-mask that are zero in current exec,
1630// Done as AND with exec. For inputs that are results of instruction that
1631// implicitly use same exec, for example compares in same basic block or SCC to
1632// VCC copy, use copy.
1635 MachineInstr *MI = MRI.getVRegDef(Reg);
1636 if (MI->getParent() != MBB)
1637 return false;
1638
1639 // Lane mask generated by SCC to VCC copy.
1640 if (MI->getOpcode() == AMDGPU::COPY) {
1641 auto DstRB = MRI.getRegBankOrNull(MI->getOperand(0).getReg());
1642 auto SrcRB = MRI.getRegBankOrNull(MI->getOperand(1).getReg());
1643 if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
1644 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1645 return true;
1646 }
1647
1648 // Lane mask generated by SCC to VCC copy
1649 if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1650 return true;
1651
1652 // Lane mask generated using compare with same exec.
1653 if (isa<GAnyCmp>(MI))
1654 return true;
1655
1656 Register LHS, RHS;
1657 // Look through AND.
1658 if (mi_match(Reg, MRI, m_GAnd(m_Reg(LHS), m_Reg(RHS))))
1659 return isLaneMaskFromSameBlock(LHS, MRI, MBB) ||
1661
1662 return false;
1663}
1664
1665bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1666 MachineBasicBlock *BB = I.getParent();
1667 const DebugLoc &DL = I.getDebugLoc();
1668 Register DstReg = I.getOperand(0).getReg();
1669 Register SrcReg = I.getOperand(2).getReg();
1670 const uint64_t BallotSize = MRI->getType(DstReg).getSizeInBits();
1671 const unsigned WaveSize = STI.getWavefrontSize();
1672
1673 // In the common case, the return type matches the wave size.
1674 // However we also support emitting i64 ballots in wave32 mode.
1675 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1676 return false;
1677
1678 std::optional<ValueAndVReg> Arg =
1680
1681 Register Dst = DstReg;
1682 // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
1683 if (BallotSize != WaveSize) {
1684 Dst = MRI->createVirtualRegister(TRI.getBoolRC());
1685 }
1686
1687 if (Arg) {
1688 const int64_t Value = Arg->Value.getZExtValue();
1689 if (Value == 0) {
1690 // Dst = S_MOV 0
1691 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1692 BuildMI(*BB, &I, DL, TII.get(Opcode), Dst).addImm(0);
1693 } else {
1694 // Dst = COPY EXEC
1695 assert(Value == 1);
1696 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst).addReg(TRI.getExec());
1697 }
1698 if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI))
1699 return false;
1700 } else {
1701 if (isLaneMaskFromSameBlock(SrcReg, *MRI, BB)) {
1702 // Dst = COPY SrcReg
1703 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst).addReg(SrcReg);
1704 if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI))
1705 return false;
1706 } else {
1707 // Dst = S_AND SrcReg, EXEC
1708 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1709 auto And = BuildMI(*BB, &I, DL, TII.get(AndOpc), Dst)
1710 .addReg(SrcReg)
1711 .addReg(TRI.getExec())
1712 .setOperandDead(3); // Dead scc
1713 constrainSelectedInstRegOperands(*And, TII, TRI, RBI);
1714 }
1715 }
1716
1717 // i64 ballot on Wave32: zero-extend i32 ballot to i64.
1718 if (BallotSize != WaveSize) {
1719 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1720 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg).addImm(0);
1721 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
1722 .addReg(Dst)
1723 .addImm(AMDGPU::sub0)
1724 .addReg(HiReg)
1725 .addImm(AMDGPU::sub1);
1726 }
1727
1728 I.eraseFromParent();
1729 return true;
1730}
1731
1732bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1733 Register DstReg = I.getOperand(0).getReg();
1734 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
1735 const TargetRegisterClass *DstRC = TRI.getRegClassForSizeOnBank(32, *DstBank);
1736 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
1737 return false;
1738
1739 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1740
1741 Module *M = MF->getFunction().getParent();
1742 const MDNode *Metadata = I.getOperand(2).getMetadata();
1743 auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString();
1744 auto *RelocSymbol = cast<GlobalVariable>(
1745 M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
1746
1747 MachineBasicBlock *BB = I.getParent();
1748 BuildMI(*BB, &I, I.getDebugLoc(),
1749 TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg)
1751
1752 I.eraseFromParent();
1753 return true;
1754}
1755
1756bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1757 Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1758
1759 Register DstReg = I.getOperand(0).getReg();
1760 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
1761 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1762 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1763
1764 MachineBasicBlock *MBB = I.getParent();
1765 const DebugLoc &DL = I.getDebugLoc();
1766
1767 auto MIB = BuildMI(*MBB, &I, DL, TII.get(Mov), DstReg);
1768
1769 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
1770 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1771 MIB.addImm(MFI->getLDSSize());
1772 } else {
1773 Module *M = MF->getFunction().getParent();
1774 const GlobalValue *GV =
1775 Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_groupstaticsize);
1777 }
1778
1779 I.eraseFromParent();
1780 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
1781 return true;
1782}
1783
1784bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1785 MachineBasicBlock *MBB = I.getParent();
1787 const DebugLoc &DL = I.getDebugLoc();
1788
1789 Register DstReg = I.getOperand(0).getReg();
1790 unsigned Depth = static_cast<unsigned>(I.getOperand(2).getImm());
1791
1792 const TargetRegisterClass *RC =
1793 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
1794 if (!RC->hasSubClassEq(&AMDGPU::SGPR_64RegClass) ||
1795 !RBI.constrainGenericRegister(DstReg, *RC, *MRI))
1796 return false;
1797
1798 // Check for kernel and shader functions
1799 if (Depth != 0 ||
1800 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1801 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg)
1802 .addImm(0);
1803 I.eraseFromParent();
1804 return true;
1805 }
1806
1807 MachineFrameInfo &MFI = MF.getFrameInfo();
1808 // There is a call to @llvm.returnaddress in this function
1809 MFI.setReturnAddressIsTaken(true);
1810
1811 // Get the return address reg and mark it as an implicit live-in
1812 Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1813 Register LiveIn = getFunctionLiveInPhysReg(MF, TII, ReturnAddrReg,
1814 AMDGPU::SReg_64RegClass, DL);
1815 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), DstReg)
1816 .addReg(LiveIn);
1817 I.eraseFromParent();
1818 return true;
1819}
1820
1821bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1822 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1823 // SelectionDAG uses for wave32 vs wave64.
1824 MachineBasicBlock *BB = MI.getParent();
1825 BuildMI(*BB, &MI, MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF))
1826 .add(MI.getOperand(1))
1827 .setOperandDead(2); // implicit-def $scc
1828
1829 Register Reg = MI.getOperand(1).getReg();
1830 MI.eraseFromParent();
1831
1832 if (!MRI->getRegClassOrNull(Reg))
1833 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass());
1834 return true;
1835}
1836
1837bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1838 MachineInstr &MI, Intrinsic::ID IntrID) const {
1839 MachineBasicBlock *MBB = MI.getParent();
1841 const DebugLoc &DL = MI.getDebugLoc();
1842
1843 unsigned IndexOperand = static_cast<unsigned>(MI.getOperand(7).getImm());
1844 bool WaveRelease = MI.getOperand(8).getImm() != 0;
1845 bool WaveDone = MI.getOperand(9).getImm() != 0;
1846
1847 if (WaveDone && !WaveRelease) {
1848 // TODO: Move this to IR verifier
1849 const Function &Fn = MF->getFunction();
1850 Fn.getContext().diagnose(DiagnosticInfoUnsupported(
1851 Fn, "ds_ordered_count: wave_done requires wave_release", DL));
1852 }
1853
1854 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1855 IndexOperand &= ~0x3f;
1856 unsigned CountDw = 0;
1857
1858 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1859 CountDw = (IndexOperand >> 24) & 0xf;
1860 IndexOperand &= ~(0xf << 24);
1861
1862 if (CountDw < 1 || CountDw > 4) {
1863 const Function &Fn = MF->getFunction();
1864 Fn.getContext().diagnose(DiagnosticInfoUnsupported(
1865 Fn, "ds_ordered_count: dword count must be between 1 and 4", DL));
1866 CountDw = 1;
1867 }
1868 }
1869
1870 if (IndexOperand) {
1871 const Function &Fn = MF->getFunction();
1872 Fn.getContext().diagnose(DiagnosticInfoUnsupported(
1873 Fn, "ds_ordered_count: bad index operand", DL));
1874 }
1875
1876 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1877 unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(*MF);
1878
1879 unsigned Offset0 = OrderedCountIndex << 2;
1880 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1881
1882 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1883 Offset1 |= (CountDw - 1) << 6;
1884
1885 if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1886 Offset1 |= ShaderType << 2;
1887
1888 unsigned Offset = Offset0 | (Offset1 << 8);
1889
1890 Register M0Val = MI.getOperand(2).getReg();
1891 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1892 .addReg(M0Val);
1893
1894 Register DstReg = MI.getOperand(0).getReg();
1895 Register ValReg = MI.getOperand(3).getReg();
1896 MachineInstrBuilder DS =
1897 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg)
1898 .addReg(ValReg)
1899 .addImm(Offset)
1900 .cloneMemRefs(MI);
1901
1902 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI))
1903 return false;
1904
1905 constrainSelectedInstRegOperands(*DS, TII, TRI, RBI);
1906 MI.eraseFromParent();
1907 return true;
1908}
1909
1910static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1911 switch (IntrID) {
1912 case Intrinsic::amdgcn_ds_gws_init:
1913 return AMDGPU::DS_GWS_INIT;
1914 case Intrinsic::amdgcn_ds_gws_barrier:
1915 return AMDGPU::DS_GWS_BARRIER;
1916 case Intrinsic::amdgcn_ds_gws_sema_v:
1917 return AMDGPU::DS_GWS_SEMA_V;
1918 case Intrinsic::amdgcn_ds_gws_sema_br:
1919 return AMDGPU::DS_GWS_SEMA_BR;
1920 case Intrinsic::amdgcn_ds_gws_sema_p:
1921 return AMDGPU::DS_GWS_SEMA_P;
1922 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1923 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1924 default:
1925 llvm_unreachable("not a gws intrinsic");
1926 }
1927}
1928
1929bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1930 Intrinsic::ID IID) const {
1931 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1932 !STI.hasGWSSemaReleaseAll()))
1933 return false;
1934
1935 // intrinsic ID, vsrc, offset
1936 const bool HasVSrc = MI.getNumOperands() == 3;
1937 assert(HasVSrc || MI.getNumOperands() == 2);
1938
1939 Register BaseOffset = MI.getOperand(HasVSrc ? 2 : 1).getReg();
1940 const RegisterBank *OffsetRB = RBI.getRegBank(BaseOffset, *MRI, TRI);
1941 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
1942 return false;
1943
1944 MachineInstr *OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI);
1945 unsigned ImmOffset;
1946
1947 MachineBasicBlock *MBB = MI.getParent();
1948 const DebugLoc &DL = MI.getDebugLoc();
1949
1950 MachineInstr *Readfirstlane = nullptr;
1951
1952 // If we legalized the VGPR input, strip out the readfirstlane to analyze the
1953 // incoming offset, in case there's an add of a constant. We'll have to put it
1954 // back later.
1955 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1956 Readfirstlane = OffsetDef;
1957 BaseOffset = OffsetDef->getOperand(1).getReg();
1958 OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI);
1959 }
1960
1961 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
1962 // If we have a constant offset, try to use the 0 in m0 as the base.
1963 // TODO: Look into changing the default m0 initialization value. If the
1964 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
1965 // the immediate offset.
1966
1967 ImmOffset = static_cast<unsigned>(
1968 OffsetDef->getOperand(1).getCImm()->getZExtValue());
1969 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0)
1970 .addImm(0);
1971 } else {
1972 std::tie(BaseOffset, ImmOffset) =
1973 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset, VT);
1974
1975 if (Readfirstlane) {
1976 // We have the constant offset now, so put the readfirstlane back on the
1977 // variable component.
1978 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI))
1979 return false;
1980
1981 Readfirstlane->getOperand(1).setReg(BaseOffset);
1982 BaseOffset = Readfirstlane->getOperand(0).getReg();
1983 } else {
1984 if (!RBI.constrainGenericRegister(BaseOffset,
1985 AMDGPU::SReg_32RegClass, *MRI))
1986 return false;
1987 }
1988
1989 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1990 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_LSHL_B32), M0Base)
1991 .addReg(BaseOffset)
1992 .addImm(16)
1993 .setOperandDead(3); // Dead scc
1994
1995 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1996 .addReg(M0Base);
1997 }
1998
1999 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2000 // offset field) % 64. Some versions of the programming guide omit the m0
2001 // part, or claim it's from offset 0.
2002
2003 unsigned Opc = gwsIntrinToOpcode(IID);
2004 const MCInstrDesc &InstrDesc = TII.get(Opc);
2005
2006 if (HasVSrc) {
2007 Register VSrc = MI.getOperand(1).getReg();
2008
2009 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
2010 const TargetRegisterClass *DataRC = TII.getRegClass(InstrDesc, Data0Idx);
2011 const TargetRegisterClass *SubRC =
2012 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2013
2014 if (!SubRC) {
2015 // 32-bit normal case.
2016 if (!RBI.constrainGenericRegister(VSrc, *DataRC, *MRI))
2017 return false;
2018
2019 BuildMI(*MBB, &MI, DL, InstrDesc)
2020 .addReg(VSrc)
2021 .addImm(ImmOffset)
2022 .cloneMemRefs(MI);
2023 } else {
2024 // Requires even register alignment, so create 64-bit value and pad the
2025 // top half with undef.
2026 Register DataReg = MRI->createVirtualRegister(DataRC);
2027 if (!RBI.constrainGenericRegister(VSrc, *SubRC, *MRI))
2028 return false;
2029
2030 Register UndefReg = MRI->createVirtualRegister(SubRC);
2031 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
2032 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), DataReg)
2033 .addReg(VSrc)
2034 .addImm(AMDGPU::sub0)
2035 .addReg(UndefReg)
2036 .addImm(AMDGPU::sub1);
2037
2038 BuildMI(*MBB, &MI, DL, InstrDesc)
2039 .addReg(DataReg)
2040 .addImm(ImmOffset)
2041 .cloneMemRefs(MI);
2042 }
2043 } else {
2044 BuildMI(*MBB, &MI, DL, InstrDesc)
2045 .addImm(ImmOffset)
2046 .cloneMemRefs(MI);
2047 }
2048
2049 MI.eraseFromParent();
2050 return true;
2051}
2052
2053bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
2054 bool IsAppend) const {
2055 Register PtrBase = MI.getOperand(2).getReg();
2056 LLT PtrTy = MRI->getType(PtrBase);
2057 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2058
2059 int64_t Offset;
2060 std::tie(PtrBase, Offset) = selectDS1Addr1OffsetImpl(MI.getOperand(2));
2061
2062 // TODO: Should this try to look through readfirstlane like GWS?
2063 if (!isDSOffsetLegal(PtrBase, Offset)) {
2064 PtrBase = MI.getOperand(2).getReg();
2065 Offset = 0;
2066 }
2067
2068 MachineBasicBlock *MBB = MI.getParent();
2069 const DebugLoc &DL = MI.getDebugLoc();
2070 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2071
2072 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
2073 .addReg(PtrBase);
2074 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI))
2075 return false;
2076
2077 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), MI.getOperand(0).getReg())
2078 .addImm(Offset)
2079 .addImm(IsGDS ? -1 : 0)
2080 .cloneMemRefs(MI);
2081 MI.eraseFromParent();
2082 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
2083 return true;
2084}
2085
2086bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
2087 MachineFunction *MF = MI.getMF();
2088 SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
2089
2090 MFInfo->setInitWholeWave();
2091 return selectImpl(MI, *CoverageInfo);
2092}
2093
2094static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
2095 bool &IsTexFail) {
2096 if (TexFailCtrl)
2097 IsTexFail = true;
2098
2099 TFE = TexFailCtrl & 0x1;
2100 TexFailCtrl &= ~(uint64_t)0x1;
2101 LWE = TexFailCtrl & 0x2;
2102 TexFailCtrl &= ~(uint64_t)0x2;
2103
2104 return TexFailCtrl == 0;
2105}
2106
2107bool AMDGPUInstructionSelector::selectImageIntrinsic(
2108 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
2109 MachineBasicBlock *MBB = MI.getParent();
2110 const DebugLoc &DL = MI.getDebugLoc();
2111 unsigned IntrOpcode = Intr->BaseOpcode;
2112
2113 // For image atomic: use no-return opcode if result is unused.
2114 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode) {
2115 Register ResultDef = MI.getOperand(0).getReg();
2116 if (MRI->use_nodbg_empty(ResultDef))
2117 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
2118 }
2119
2120 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2122
2123 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim);
2124 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
2125 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
2126 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
2127 const bool IsGFX13Plus = AMDGPU::isGFX13Plus(STI);
2128
2129 const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
2130
2131 Register VDataIn = AMDGPU::NoRegister;
2132 Register VDataOut = AMDGPU::NoRegister;
2133 LLT VDataTy;
2134 int NumVDataDwords = -1;
2135 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2136 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2137
2138 bool Unorm;
2139 if (!BaseOpcode->Sampler)
2140 Unorm = true;
2141 else
2142 Unorm = MI.getOperand(ArgOffset + Intr->UnormIndex).getImm() != 0;
2143
2144 bool TFE;
2145 bool LWE;
2146 bool IsTexFail = false;
2147 if (!parseTexFail(MI.getOperand(ArgOffset + Intr->TexFailCtrlIndex).getImm(),
2148 TFE, LWE, IsTexFail))
2149 return false;
2150
2151 const int64_t Flags = MI.getOperand(ArgOffset + Intr->NumArgs).getImm();
2152 const bool IsA16 = (Flags & 1) != 0;
2153 const bool IsG16 = (Flags & 2) != 0;
2154
2155 // A16 implies 16 bit gradients if subtarget doesn't support G16
2156 if (IsA16 && !STI.hasG16() && !IsG16)
2157 return false;
2158
2159 unsigned DMask = 0;
2160 unsigned DMaskLanes = 0;
2161
2162 if (BaseOpcode->Atomic) {
2163 if (!BaseOpcode->NoReturn)
2164 VDataOut = MI.getOperand(0).getReg();
2165 VDataIn = MI.getOperand(2).getReg();
2166 LLT Ty = MRI->getType(VDataIn);
2167
2168 // Be careful to allow atomic swap on 16-bit element vectors.
2169 const bool Is64Bit = BaseOpcode->AtomicX2 ?
2170 Ty.getSizeInBits() == 128 :
2171 Ty.getSizeInBits() == 64;
2172
2173 if (BaseOpcode->AtomicX2) {
2174 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2175
2176 DMask = Is64Bit ? 0xf : 0x3;
2177 NumVDataDwords = Is64Bit ? 4 : 2;
2178 } else {
2179 DMask = Is64Bit ? 0x3 : 0x1;
2180 NumVDataDwords = Is64Bit ? 2 : 1;
2181 }
2182 } else {
2183 DMask = static_cast<unsigned>(
2184 MI.getOperand(ArgOffset + Intr->DMaskIndex).getImm());
2185 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(DMask);
2186
2187 if (BaseOpcode->Store) {
2188 VDataIn = MI.getOperand(1).getReg();
2189 VDataTy = MRI->getType(VDataIn);
2190 NumVDataDwords = static_cast<int>((VDataTy.getSizeInBits() + 31) / 32);
2191 } else if (BaseOpcode->NoReturn) {
2192 NumVDataDwords = 0;
2193 } else {
2194 VDataOut = MI.getOperand(0).getReg();
2195 VDataTy = MRI->getType(VDataOut);
2196 NumVDataDwords = DMaskLanes;
2197
2198 if (IsD16 && !STI.hasUnpackedD16VMem())
2199 NumVDataDwords = (DMaskLanes + 1) / 2;
2200 }
2201 }
2202
2203 // Set G16 opcode
2204 if (Subtarget->hasG16() && IsG16) {
2205 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2207 assert(G16MappingInfo);
2208 IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
2209 }
2210
2211 // TODO: Check this in verifier.
2212 assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
2213
2214 unsigned CPol = static_cast<unsigned>(
2215 MI.getOperand(ArgOffset + Intr->CachePolicyIndex).getImm());
2216 // Keep GLC only when the atomic's result is actually used.
2217 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
2219 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
2221 return false;
2222
2223 int NumVAddrRegs = 0;
2224 int NumVAddrDwords = 0;
2225 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
2226 // Skip the $noregs and 0s inserted during legalization.
2227 MachineOperand &AddrOp = MI.getOperand(ArgOffset + I);
2228 if (!AddrOp.isReg())
2229 continue; // XXX - Break?
2230
2231 Register Addr = AddrOp.getReg();
2232 if (!Addr)
2233 break;
2234
2235 ++NumVAddrRegs;
2236 NumVAddrDwords +=
2237 static_cast<int>((MRI->getType(Addr).getSizeInBits() + 31) / 32);
2238 }
2239
2240 // The legalizer preprocessed the intrinsic arguments. If we aren't using
2241 // NSA, these should have been packed into a single value in the first
2242 // address register
2243 const bool UseNSA =
2244 NumVAddrRegs != 1 &&
2245 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2246 : NumVAddrDwords == NumVAddrRegs);
2247 if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) {
2248 LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
2249 return false;
2250 }
2251
2252 if (IsTexFail)
2253 ++NumVDataDwords;
2254
2255 int Opcode = -1;
2256 if (IsGFX13Plus) {
2257 Register RsrcReg = MI.getOperand(ArgOffset + Intr->RsrcIndex).getReg();
2258 const LLT I32 = LLT::integer(32);
2259 bool IndexedRsrc = MRI->getType(RsrcReg) == I32;
2260 bool IndexedSamp = false;
2261 if (BaseOpcode->Sampler) {
2262 Register SampReg = MI.getOperand(ArgOffset + Intr->SampIndex).getReg();
2263 IndexedSamp = MRI->getType(SampReg) == I32;
2264 }
2265 Opcode =
2266 AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx13, NumVDataDwords,
2267 NumVAddrDwords, IndexedRsrc, IndexedSamp);
2268 } else if (IsGFX12Plus) {
2269 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx12,
2270 NumVDataDwords, NumVAddrDwords);
2271 } else if (IsGFX11Plus) {
2272 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode,
2273 UseNSA ? AMDGPU::MIMGEncGfx11NSA
2274 : AMDGPU::MIMGEncGfx11Default,
2275 NumVDataDwords, NumVAddrDwords);
2276 } else if (IsGFX10Plus) {
2277 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode,
2278 UseNSA ? AMDGPU::MIMGEncGfx10NSA
2279 : AMDGPU::MIMGEncGfx10Default,
2280 NumVDataDwords, NumVAddrDwords);
2281 } else {
2282 if (Subtarget->hasGFX90AInsts()) {
2283 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a,
2284 NumVDataDwords, NumVAddrDwords);
2285 if (Opcode == -1) {
2286 LLVM_DEBUG(
2287 dbgs()
2288 << "requested image instruction is not supported on this GPU\n");
2289 return false;
2290 }
2291 }
2292 if (Opcode == -1 &&
2293 STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
2294 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8,
2295 NumVDataDwords, NumVAddrDwords);
2296 if (Opcode == -1)
2297 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6,
2298 NumVDataDwords, NumVAddrDwords);
2299 }
2300 if (Opcode == -1)
2301 return false;
2302
2303 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opcode))
2304 .cloneMemRefs(MI);
2305
2306 if (VDataOut) {
2307 if (BaseOpcode->AtomicX2) {
2308 const bool Is64 = MRI->getType(VDataOut).getSizeInBits() == 64;
2309
2310 Register TmpReg = MRI->createVirtualRegister(
2311 Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2312 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2313
2314 MIB.addDef(TmpReg);
2315 if (!MRI->use_empty(VDataOut)) {
2316 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), VDataOut)
2317 .addReg(TmpReg, {}, SubReg);
2318 }
2319
2320 } else {
2321 MIB.addDef(VDataOut); // vdata output
2322 }
2323 }
2324
2325 if (VDataIn)
2326 MIB.addReg(VDataIn); // vdata input
2327
2328 for (int I = 0; I != NumVAddrRegs; ++I) {
2329 MachineOperand &SrcOp = MI.getOperand(ArgOffset + Intr->VAddrStart + I);
2330 if (SrcOp.isReg()) {
2331 assert(SrcOp.getReg() != 0);
2332 MIB.addReg(SrcOp.getReg());
2333 }
2334 }
2335
2336 MIB.addReg(MI.getOperand(ArgOffset + Intr->RsrcIndex).getReg());
2337 if (BaseOpcode->Sampler)
2338 MIB.addReg(MI.getOperand(ArgOffset + Intr->SampIndex).getReg());
2339
2340 MIB.addImm(DMask); // dmask
2341
2342 if (IsGFX10Plus)
2343 MIB.addImm(DimInfo->Encoding);
2344 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::unorm))
2345 MIB.addImm(Unorm);
2346
2347 MIB.addImm(CPol);
2348 MIB.addImm(IsA16 && // a16 or r128
2349 STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0);
2350 if (IsGFX10Plus)
2351 MIB.addImm(IsA16 ? -1 : 0);
2352
2353 if (!Subtarget->hasGFX90AInsts()) {
2354 MIB.addImm(TFE); // tfe
2355 } else if (TFE) {
2356 LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n");
2357 return false;
2358 }
2359
2360 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::lwe))
2361 MIB.addImm(LWE); // lwe
2362 if (!IsGFX10Plus)
2363 MIB.addImm(DimInfo->DA ? -1 : 0);
2364 if (BaseOpcode->HasD16)
2365 MIB.addImm(IsD16 ? -1 : 0);
2366
2367 MI.eraseFromParent();
2368 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
2369 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr);
2370 return true;
2371}
2372
2373// We need to handle this here because tablegen doesn't support matching
2374// instructions with multiple outputs.
2375bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2376 MachineInstr &MI) const {
2377 Register Dst0 = MI.getOperand(0).getReg();
2378 Register Dst1 = MI.getOperand(1).getReg();
2379
2380 const DebugLoc &DL = MI.getDebugLoc();
2381 MachineBasicBlock *MBB = MI.getParent();
2382
2383 Register Addr = MI.getOperand(3).getReg();
2384 Register Data0 = MI.getOperand(4).getReg();
2385 Register Data1 = MI.getOperand(5).getReg();
2386 unsigned Offset = static_cast<unsigned>(MI.getOperand(6).getImm());
2387
2388 unsigned Opc;
2389 switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
2390 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2391 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2392 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2393 break;
2394 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2395 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2396 break;
2397 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2398 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2399 break;
2400 }
2401
2402 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc), Dst0)
2403 .addDef(Dst1)
2404 .addUse(Addr)
2405 .addUse(Data0)
2406 .addUse(Data1)
2407 .addImm(Offset)
2408 .cloneMemRefs(MI);
2409
2410 MI.eraseFromParent();
2411 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
2412 return true;
2413}
2414
2415bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2416 MachineInstr &I) const {
2417 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(I).getIntrinsicID();
2418 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
2419 AMDGPU::lookupBufferIntrinsic(IntrinsicID);
2420 BufInfo && BufInfo->Kind == AMDGPU::BufferKindLoadToLDS)
2421 return selectBufferLoadLds(I, *BufInfo);
2422
2423 switch (IntrinsicID) {
2424 case Intrinsic::amdgcn_end_cf:
2425 return selectEndCfIntrinsic(I);
2426 case Intrinsic::amdgcn_ds_ordered_add:
2427 case Intrinsic::amdgcn_ds_ordered_swap:
2428 return selectDSOrderedIntrinsic(I, IntrinsicID);
2429 case Intrinsic::amdgcn_ds_gws_init:
2430 case Intrinsic::amdgcn_ds_gws_barrier:
2431 case Intrinsic::amdgcn_ds_gws_sema_v:
2432 case Intrinsic::amdgcn_ds_gws_sema_br:
2433 case Intrinsic::amdgcn_ds_gws_sema_p:
2434 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2435 return selectDSGWSIntrinsic(I, IntrinsicID);
2436 case Intrinsic::amdgcn_ds_append:
2437 return selectDSAppendConsume(I, true);
2438 case Intrinsic::amdgcn_ds_consume:
2439 return selectDSAppendConsume(I, false);
2440 case Intrinsic::amdgcn_init_whole_wave:
2441 return selectInitWholeWave(I);
2442 // Until we can store both the address space of the global and the LDS
2443 // arguments by having tto MachineMemOperands on an intrinsic, we just trust
2444 // that the argument is a global pointer (buffer pointers have been handled by
2445 // a LLVM IR-level lowering).
2446 case Intrinsic::amdgcn_load_to_lds:
2447 case Intrinsic::amdgcn_load_async_to_lds:
2448 case Intrinsic::amdgcn_global_load_lds:
2449 case Intrinsic::amdgcn_global_load_async_lds:
2450 return selectGlobalLoadLds(I);
2451 case Intrinsic::amdgcn_tensor_load_to_lds:
2452 case Intrinsic::amdgcn_tensor_store_from_lds:
2453 return selectTensorLoadStore(I, IntrinsicID);
2454 case Intrinsic::amdgcn_asyncmark:
2455 case Intrinsic::amdgcn_wait_asyncmark:
2456 if (!Subtarget->hasAsyncMark())
2457 return false;
2458 break;
2459 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2460 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2461 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2462 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2463 return selectDSBvhStackIntrinsic(I);
2464 case Intrinsic::amdgcn_s_alloc_vgpr: {
2465 // S_ALLOC_VGPR doesn't have a destination register, it just implicitly sets
2466 // SCC. We then need to COPY it into the result vreg.
2467 MachineBasicBlock *MBB = I.getParent();
2468 const DebugLoc &DL = I.getDebugLoc();
2469
2470 Register ResReg = I.getOperand(0).getReg();
2471
2472 MachineInstr *AllocMI = BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_ALLOC_VGPR))
2473 .add(I.getOperand(2));
2474 (void)BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), ResReg)
2475 .addReg(AMDGPU::SCC);
2476 I.eraseFromParent();
2477 constrainSelectedInstRegOperands(*AllocMI, TII, TRI, RBI);
2478 return RBI.constrainGenericRegister(ResReg, AMDGPU::SReg_32RegClass, *MRI);
2479 }
2480 case Intrinsic::amdgcn_s_barrier_init:
2481 case Intrinsic::amdgcn_s_barrier_signal_var:
2482 return selectNamedBarrierInit(I, IntrinsicID);
2483 case Intrinsic::amdgcn_s_wakeup_barrier:
2484 case Intrinsic::amdgcn_s_barrier_join:
2485 case Intrinsic::amdgcn_s_get_named_barrier_state:
2486 return selectNamedBarrierInst(I, IntrinsicID);
2487 case Intrinsic::amdgcn_s_get_barrier_state:
2488 return selectSGetBarrierState(I, IntrinsicID);
2489 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2490 return selectSBarrierSignalIsfirst(I, IntrinsicID);
2491 }
2492 return selectImpl(I, *CoverageInfo);
2493}
2494
2495bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2496 if (selectImpl(I, *CoverageInfo))
2497 return true;
2498
2499 MachineBasicBlock *BB = I.getParent();
2500 const DebugLoc &DL = I.getDebugLoc();
2501
2502 Register DstReg = I.getOperand(0).getReg();
2503 uint64_t Size = TRI.getRegSizeInBits(DstReg, *MRI);
2504 assert(Size <= 32 || Size == 64);
2505 const MachineOperand &CCOp = I.getOperand(1);
2506 Register CCReg = CCOp.getReg();
2507 if (!isVCC(CCReg, *MRI)) {
2508 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2509 AMDGPU::S_CSELECT_B32;
2510 MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
2511 .addReg(CCReg);
2512
2513 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
2514 // bank, because it does not cover the register class that we used to represent
2515 // for it. So we need to manually set the register class here.
2516 if (!MRI->getRegClassOrNull(CCReg))
2517 MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForReg(CCReg, *MRI));
2518 MachineInstr *Select = BuildMI(*BB, &I, DL, TII.get(SelectOpcode), DstReg)
2519 .add(I.getOperand(2))
2520 .add(I.getOperand(3));
2521
2523 constrainSelectedInstRegOperands(*CopySCC, TII, TRI, RBI);
2524 I.eraseFromParent();
2525 return true;
2526 }
2527
2528 // Wide VGPR select should have been split in RegBankSelect.
2529 if (Size > 32)
2530 return false;
2531
2532 MachineInstr *Select =
2533 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
2534 .addImm(0)
2535 .add(I.getOperand(3))
2536 .addImm(0)
2537 .add(I.getOperand(2))
2538 .add(I.getOperand(1));
2539
2541 I.eraseFromParent();
2542 return true;
2543}
2544
2545bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2546 Register DstReg = I.getOperand(0).getReg();
2547 Register SrcReg = I.getOperand(1).getReg();
2548 const LLT DstTy = MRI->getType(DstReg);
2549 const LLT SrcTy = MRI->getType(SrcReg);
2550 const LLT S1 = LLT::scalar(1);
2551
2552 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
2553 const RegisterBank *DstRB;
2554 if (DstTy == S1) {
2555 // This is a special case. We don't treat s1 for legalization artifacts as
2556 // vcc booleans.
2557 DstRB = SrcRB;
2558 } else {
2559 DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
2560 if (SrcRB != DstRB)
2561 return false;
2562 }
2563
2564 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2565
2566 uint64_t DstSize = DstTy.getSizeInBits();
2567 uint64_t SrcSize = SrcTy.getSizeInBits();
2568
2569 const TargetRegisterClass *SrcRC =
2570 TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB);
2571 const TargetRegisterClass *DstRC =
2572 TRI.getRegClassForSizeOnBank(DstSize, *DstRB);
2573 if (!SrcRC || !DstRC)
2574 return false;
2575
2576 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) ||
2577 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
2578 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
2579 return false;
2580 }
2581
2582 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2583 assert(STI.useRealTrue16Insts());
2584 const DebugLoc &DL = I.getDebugLoc();
2585 MachineBasicBlock *MBB = I.getParent();
2586 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), DstReg)
2587 .addReg(SrcReg, {}, AMDGPU::lo16);
2588 I.eraseFromParent();
2589 return true;
2590 }
2591
2592 if (DstTy == LLT::fixed_vector(2, 16) && SrcTy == LLT::fixed_vector(2, 32)) {
2593 MachineBasicBlock *MBB = I.getParent();
2594 const DebugLoc &DL = I.getDebugLoc();
2595
2596 Register LoReg = MRI->createVirtualRegister(DstRC);
2597 Register HiReg = MRI->createVirtualRegister(DstRC);
2598 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), LoReg)
2599 .addReg(SrcReg, {}, AMDGPU::sub0);
2600 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), HiReg)
2601 .addReg(SrcReg, {}, AMDGPU::sub1);
2602
2603 if (IsVALU && STI.hasSDWA()) {
2604 // Write the low 16-bits of the high element into the high 16-bits of the
2605 // low element.
2606 MachineInstr *MovSDWA =
2607 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg)
2608 .addImm(0) // $src0_modifiers
2609 .addReg(HiReg) // $src0
2610 .addImm(0) // $clamp
2611 .addImm(AMDGPU::SDWA::WORD_1) // $dst_sel
2612 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2613 .addImm(AMDGPU::SDWA::WORD_0) // $src0_sel
2614 .addReg(LoReg, RegState::Implicit);
2615 MovSDWA->tieOperands(0, MovSDWA->getNumOperands() - 1);
2616 } else {
2617 Register TmpReg0 = MRI->createVirtualRegister(DstRC);
2618 Register TmpReg1 = MRI->createVirtualRegister(DstRC);
2619 Register ImmReg = MRI->createVirtualRegister(DstRC);
2620 if (IsVALU) {
2621 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), TmpReg0)
2622 .addImm(16)
2623 .addReg(HiReg);
2624 } else {
2625 BuildMI(*MBB, I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0)
2626 .addReg(HiReg)
2627 .addImm(16)
2628 .setOperandDead(3); // Dead scc
2629 }
2630
2631 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2632 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2633 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2634
2635 BuildMI(*MBB, I, DL, TII.get(MovOpc), ImmReg)
2636 .addImm(0xffff);
2637 auto And = BuildMI(*MBB, I, DL, TII.get(AndOpc), TmpReg1)
2638 .addReg(LoReg)
2639 .addReg(ImmReg);
2640 auto Or = BuildMI(*MBB, I, DL, TII.get(OrOpc), DstReg)
2641 .addReg(TmpReg0)
2642 .addReg(TmpReg1);
2643
2644 if (!IsVALU) {
2645 And.setOperandDead(3); // Dead scc
2646 Or.setOperandDead(3); // Dead scc
2647 }
2648 }
2649
2650 I.eraseFromParent();
2651 return true;
2652 }
2653
2654 if (!DstTy.isScalar())
2655 return false;
2656
2657 if (SrcSize > 32) {
2658 unsigned SubRegIdx = DstSize < 32
2659 ? static_cast<unsigned>(AMDGPU::sub0)
2660 : TRI.getSubRegFromChannel(0, DstSize / 32);
2661 if (SubRegIdx == AMDGPU::NoSubRegister)
2662 return false;
2663
2664 // Deal with weird cases where the class only partially supports the subreg
2665 // index.
2666 const TargetRegisterClass *SrcWithSubRC
2667 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2668 if (!SrcWithSubRC)
2669 return false;
2670
2671 if (SrcWithSubRC != SrcRC) {
2672 if (!RBI.constrainGenericRegister(SrcReg, *SrcWithSubRC, *MRI))
2673 return false;
2674 }
2675
2676 I.getOperand(1).setSubReg(SubRegIdx);
2677 }
2678
2679 I.setDesc(TII.get(TargetOpcode::COPY));
2680 return true;
2681}
2682
2683/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2684static bool shouldUseAndMask(uint64_t Size, unsigned &Mask) {
2685 Mask = maskTrailingOnes<unsigned>(static_cast<unsigned>(Size));
2686 int SignedMask = static_cast<int>(Mask);
2687 return SignedMask >= -16 && SignedMask <= 64;
2688}
2689
2690// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2691const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2693 const TargetRegisterInfo &TRI) const {
2694 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2695 if (auto *RB = dyn_cast<const RegisterBank *>(RegClassOrBank))
2696 return RB;
2697
2698 // Ignore the type, since we don't use vcc in artifacts.
2699 if (auto *RC = dyn_cast<const TargetRegisterClass *>(RegClassOrBank))
2700 return &RBI.getRegBankFromRegClass(*RC, LLT());
2701 return nullptr;
2702}
2703
2704bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2705 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2706 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg;
2707 const DebugLoc &DL = I.getDebugLoc();
2708 MachineBasicBlock &MBB = *I.getParent();
2709 const Register DstReg = I.getOperand(0).getReg();
2710 const Register SrcReg = I.getOperand(1).getReg();
2711
2712 const LLT DstTy = MRI->getType(DstReg);
2713 const LLT SrcTy = MRI->getType(SrcReg);
2714 const uint64_t SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG
2715 ? I.getOperand(2).getImm()
2716 : SrcTy.getSizeInBits();
2717 const uint64_t DstSize = DstTy.getSizeInBits();
2718 if (!DstTy.isScalar())
2719 return false;
2720
2721 // Artifact casts should never use vcc.
2722 const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI);
2723
2724 // FIXME: This should probably be illegal and split earlier.
2725 if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2726 if (DstSize <= 32)
2727 return selectCOPY(I);
2728
2729 const TargetRegisterClass *SrcRC =
2730 TRI.getRegClassForTypeOnBank(SrcTy, *SrcBank);
2731 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
2732 const TargetRegisterClass *DstRC =
2733 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
2734
2735 Register UndefReg = MRI->createVirtualRegister(SrcRC);
2736 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
2737 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
2738 .addReg(SrcReg)
2739 .addImm(AMDGPU::sub0)
2740 .addReg(UndefReg)
2741 .addImm(AMDGPU::sub1);
2742 I.eraseFromParent();
2743
2744 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) &&
2745 RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI);
2746 }
2747
2748 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2749 // 64-bit should have been split up in RegBankSelect
2750
2751 // Try to use an and with a mask if it will save code size.
2752 unsigned Mask;
2753 if (!Signed && shouldUseAndMask(SrcSize, Mask)) {
2754 MachineInstr *ExtI =
2755 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg)
2756 .addImm(Mask)
2757 .addReg(SrcReg);
2758 I.eraseFromParent();
2759 constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
2760 return true;
2761 }
2762
2763 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2764 MachineInstr *ExtI =
2765 BuildMI(MBB, I, DL, TII.get(BFE), DstReg)
2766 .addReg(SrcReg)
2767 .addImm(0) // Offset
2768 .addImm(SrcSize); // Width
2769 I.eraseFromParent();
2770 constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
2771 return true;
2772 }
2773
2774 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2775 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2776 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2777 if (!RBI.constrainGenericRegister(SrcReg, SrcRC, *MRI))
2778 return false;
2779
2780 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2781 const unsigned SextOpc = SrcSize == 8 ?
2782 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2783 BuildMI(MBB, I, DL, TII.get(SextOpc), DstReg)
2784 .addReg(SrcReg);
2785 I.eraseFromParent();
2786 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
2787 }
2788
2789 // Using a single 32-bit SALU to calculate the high half is smaller than
2790 // S_BFE with a literal constant operand.
2791 if (DstSize > 32 && SrcSize == 32) {
2792 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2793 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2794 if (Signed) {
2795 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_ASHR_I32), HiReg)
2796 .addReg(SrcReg, {}, SubReg)
2797 .addImm(31)
2798 .setOperandDead(3); // Dead scc
2799 } else {
2800 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg)
2801 .addImm(0);
2802 }
2803 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
2804 .addReg(SrcReg, {}, SubReg)
2805 .addImm(AMDGPU::sub0)
2806 .addReg(HiReg)
2807 .addImm(AMDGPU::sub1);
2808 I.eraseFromParent();
2809 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass,
2810 *MRI);
2811 }
2812
2813 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2814 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2815
2816 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2817 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2818 // We need a 64-bit register source, but the high bits don't matter.
2819 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
2820 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2821 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2822
2823 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
2824 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg)
2825 .addReg(SrcReg, {}, SubReg)
2826 .addImm(AMDGPU::sub0)
2827 .addReg(UndefReg)
2828 .addImm(AMDGPU::sub1);
2829
2830 BuildMI(MBB, I, DL, TII.get(BFE64), DstReg)
2831 .addReg(ExtReg)
2832 .addImm(SrcSize << 16);
2833
2834 I.eraseFromParent();
2835 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI);
2836 }
2837
2838 unsigned Mask;
2839 if (!Signed && shouldUseAndMask(SrcSize, Mask)) {
2840 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg)
2841 .addReg(SrcReg)
2842 .addImm(Mask)
2843 .setOperandDead(3); // Dead scc
2844 } else {
2845 BuildMI(MBB, I, DL, TII.get(BFE32), DstReg)
2846 .addReg(SrcReg)
2847 .addImm(SrcSize << 16);
2848 }
2849
2850 I.eraseFromParent();
2851 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
2852 }
2853
2854 return false;
2855}
2856
2860
2862 Register BitcastSrc;
2863 if (mi_match(Reg, MRI, m_GBitcast(m_Reg(BitcastSrc))))
2864 Reg = BitcastSrc;
2865 return Reg;
2866}
2867
2869 Register &Out) {
2870 // When unmerging a register that is composed of 2 x 16-bit values allow to
2871 // use an extract hi instruction for the upper 16 bits. We only need to check
2872 // the size of `In` as all defs are guaranteed to be the same type for
2873 // GUnmerge.
2874 GUnmerge *Unmerge;
2875 if (mi_match(In, MRI, m_GUnmerge(Unmerge))) {
2876 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(1).getReg() == In &&
2877 MRI.getType(In).getSizeInBits() == 16) {
2878 Out = stripBitCast(Unmerge->getSourceReg(), MRI);
2879 return true;
2880 }
2881 }
2882
2884 if (!mi_match(In, MRI, m_GTrunc(m_Reg(Trunc))))
2885 return false;
2886
2887 Register LShlSrc;
2888 Register Cst;
2889 if (mi_match(Trunc, MRI, m_GLShr(m_Reg(LShlSrc), m_Reg(Cst)))) {
2890 Cst = stripCopy(Cst, MRI);
2891 if (mi_match(Cst, MRI, m_SpecificICst(16))) {
2892 Register Result = stripBitCast(LShlSrc, MRI);
2893 // op_sel can only select the high half of a 32-bit source. If the
2894 // shift source is wider (e.g. a 64-bit D16 image sample result), we
2895 // cannot fold it directly. Do not assign Out before this check since
2896 // the caller may pass the same register for both In and Out.
2897 if (MRI.getType(Result).getSizeInBits() != 32)
2898 return false;
2899 Out = Result;
2900 return true;
2901 }
2902 }
2903
2904 ArrayRef<int> Mask;
2905 Register Src1;
2906 if (!mi_match(Trunc, MRI, m_GShuffleVector(m_Reg(Src1), m_Reg(), Mask)))
2907 return false;
2908
2909 assert(MRI.getType(Src1) == LLT::fixed_vector(2, 16));
2910 assert(Mask.size() == 2);
2911
2912 if (Mask[0] == 1 && Mask[1] <= 1) {
2913 Out = Trunc;
2914 return true;
2915 }
2916
2917 return false;
2918}
2919
2921 Register &Out) {
2922 // There could be a bitcast between the extraction and its use.
2923 In = stripBitCast(In, MRI);
2924
2925 // The first def of a 2 x 16-bit unmerge is the low half of its source.
2926 if (auto *Unmerge = dyn_cast<GUnmerge>(MRI.getVRegDef(In))) {
2927 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(0).getReg() == In &&
2928 MRI.getType(In).getSizeInBits() == 16) {
2929 Out = stripBitCast(Unmerge->getSourceReg(), MRI);
2930 return true;
2931 }
2932 }
2933
2934 // A truncation from 32 to 16 bits keeps the low half in place.
2936 if (mi_match(In, MRI, m_GTrunc(m_Reg(Trunc))) &&
2937 MRI.getType(Trunc).getSizeInBits() == 32) {
2938 Out = stripBitCast(Trunc, MRI);
2939 return true;
2940 }
2941
2942 return false;
2943}
2944
2945bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const {
2946 if (!Subtarget->hasSALUFloatInsts())
2947 return false;
2948
2949 Register Dst = I.getOperand(0).getReg();
2950 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI);
2951 if (DstRB->getID() != AMDGPU::SGPRRegBankID)
2952 return false;
2953
2954 Register Src = I.getOperand(1).getReg();
2955
2956 if (MRI->getType(Dst) == LLT::scalar(32) &&
2957 MRI->getType(Src) == LLT::scalar(16)) {
2958 if (isExtractHiElt(*MRI, Src, Src)) {
2959 MachineBasicBlock *BB = I.getParent();
2960 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_CVT_HI_F32_F16), Dst)
2961 .addUse(Src);
2962 I.eraseFromParent();
2963 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI);
2964 }
2965 }
2966
2967 return false;
2968}
2969
2970bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2971 // Only manually handle the f64 SGPR case.
2972 //
2973 // FIXME: This is a workaround for 2.5 different tablegen problems. Because
2974 // the bit ops theoretically have a second result due to the implicit def of
2975 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2976 // that is easy by disabling the check. The result works, but uses a
2977 // nonsensical sreg32orlds_and_sreg_1 regclass.
2978 //
2979 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2980 // the variadic REG_SEQUENCE operands.
2981
2982 Register Dst = MI.getOperand(0).getReg();
2983 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI);
2984 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
2985 MRI->getType(Dst) != LLT::scalar(64))
2986 return false;
2987
2988 Register Src = MI.getOperand(1).getReg();
2989 MachineInstr *Fabs = getOpcodeDef(TargetOpcode::G_FABS, Src, *MRI);
2990 if (Fabs)
2991 Src = Fabs->getOperand(1).getReg();
2992
2993 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) ||
2994 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI))
2995 return false;
2996
2997 MachineBasicBlock *BB = MI.getParent();
2998 const DebugLoc &DL = MI.getDebugLoc();
2999 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3000 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3001 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3002 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3003
3004 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg)
3005 .addReg(Src, {}, AMDGPU::sub0);
3006 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg)
3007 .addReg(Src, {}, AMDGPU::sub1);
3008 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg)
3009 .addImm(0x80000000);
3010
3011 // Set or toggle sign bit.
3012 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
3013 BuildMI(*BB, &MI, DL, TII.get(Opc), OpReg)
3014 .addReg(HiReg)
3015 .addReg(ConstReg)
3016 .setOperandDead(3); // Dead scc
3017 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst)
3018 .addReg(LoReg)
3019 .addImm(AMDGPU::sub0)
3020 .addReg(OpReg)
3021 .addImm(AMDGPU::sub1);
3022 MI.eraseFromParent();
3023 return true;
3024}
3025
3026// FIXME: This is a workaround for the same tablegen problems as G_FNEG
3027bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
3028 Register Dst = MI.getOperand(0).getReg();
3029 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI);
3030 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
3031 MRI->getType(Dst) != LLT::scalar(64))
3032 return false;
3033
3034 Register Src = MI.getOperand(1).getReg();
3035 MachineBasicBlock *BB = MI.getParent();
3036 const DebugLoc &DL = MI.getDebugLoc();
3037 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3038 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3039 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3040 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3041
3042 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) ||
3043 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI))
3044 return false;
3045
3046 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg)
3047 .addReg(Src, {}, AMDGPU::sub0);
3048 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg)
3049 .addReg(Src, {}, AMDGPU::sub1);
3050 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg)
3051 .addImm(0x7fffffff);
3052
3053 // Clear sign bit.
3054 // TODO: Should this used S_BITSET0_*?
3055 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_AND_B32), OpReg)
3056 .addReg(HiReg)
3057 .addReg(ConstReg)
3058 .setOperandDead(3); // Dead scc
3059 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst)
3060 .addReg(LoReg)
3061 .addImm(AMDGPU::sub0)
3062 .addReg(OpReg)
3063 .addImm(AMDGPU::sub1);
3064
3065 MI.eraseFromParent();
3066 return true;
3067}
3068
3069static bool isConstant(const MachineInstr &MI) {
3070 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3071}
3072
3073void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
3074 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
3075
3076 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3077 const MachineInstr *PtrMI = MRI.getVRegDef(Load.getOperand(OpNo).getReg());
3078
3079 assert(PtrMI);
3080
3081 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
3082 return;
3083
3084 GEPInfo GEPInfo;
3085
3086 for (unsigned i = 1; i != 3; ++i) {
3087 const MachineOperand &GEPOp = PtrMI->getOperand(i);
3088 const MachineInstr *OpDef = MRI.getVRegDef(GEPOp.getReg());
3089 assert(OpDef);
3090 if (i == 2 && isConstant(*OpDef)) {
3091 // TODO: Could handle constant base + variable offset, but a combine
3092 // probably should have commuted it.
3093 assert(GEPInfo.Imm == 0);
3094 GEPInfo.Imm = OpDef->getOperand(1).getCImm()->getSExtValue();
3095 continue;
3096 }
3097 const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI);
3098 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
3099 GEPInfo.SgprParts.push_back(GEPOp.getReg());
3100 else
3101 GEPInfo.VgprParts.push_back(GEPOp.getReg());
3102 }
3103
3104 AddrInfo.push_back(GEPInfo);
3105 getAddrModeInfo(*PtrMI, MRI, AddrInfo);
3106}
3107
3108bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
3109 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3110}
3111
3112bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
3113 if (!MI.hasOneMemOperand())
3114 return false;
3115
3116 const MachineMemOperand *MMO = *MI.memoperands_begin();
3117 const Value *Ptr = MMO->getValue();
3118
3119 // UndefValue means this is a load of a kernel input. These are uniform.
3120 // Sometimes LDS instructions have constant pointers.
3121 // If Ptr is null, then that means this mem operand contains a
3122 // PseudoSourceValue like GOT.
3124 return true;
3125
3127 return true;
3128
3129 if (MI.getOpcode() == AMDGPU::G_PREFETCH)
3130 return RBI.getRegBank(MI.getOperand(0).getReg(), *MRI, TRI)->getID() ==
3131 AMDGPU::SGPRRegBankID;
3132
3133 const Instruction *I = dyn_cast<Instruction>(Ptr);
3134 return I && I->getMetadata("amdgpu.uniform");
3135}
3136
3137bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
3138 for (const GEPInfo &GEPInfo : AddrInfo) {
3139 if (!GEPInfo.VgprParts.empty())
3140 return true;
3141 }
3142 return false;
3143}
3144
3145void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
3146 const LLT PtrTy = MRI->getType(I.getOperand(1).getReg());
3147 unsigned AS = PtrTy.getAddressSpace();
3149 STI.ldsRequiresM0Init()) {
3150 MachineBasicBlock *BB = I.getParent();
3151
3152 // If DS instructions require M0 initialization, insert it before selecting.
3153 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3154 .addImm(-1);
3155 }
3156}
3157
3158bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3159 MachineInstr &I) const {
3160 initM0(I);
3161 return selectImpl(I, *CoverageInfo);
3162}
3163
3165 if (Reg.isPhysical())
3166 return false;
3167
3168 MachineInstr &MI = *MRI.getVRegDef(Reg);
3169 const unsigned Opcode = MI.getOpcode();
3170
3171 if (Opcode == AMDGPU::COPY)
3172 return isVCmpResult(MI.getOperand(1).getReg(), MRI);
3173
3174 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3175 Opcode == AMDGPU::G_XOR)
3176 return isVCmpResult(MI.getOperand(1).getReg(), MRI) &&
3177 isVCmpResult(MI.getOperand(2).getReg(), MRI);
3178
3179 if (auto *GI = dyn_cast<GIntrinsic>(&MI))
3180 return GI->is(Intrinsic::amdgcn_class);
3181
3182 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3183}
3184
3185bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
3186 MachineBasicBlock *BB = I.getParent();
3187 MachineOperand &CondOp = I.getOperand(0);
3188 Register CondReg = CondOp.getReg();
3189 const DebugLoc &DL = I.getDebugLoc();
3190
3191 unsigned BrOpcode;
3192 Register CondPhysReg;
3193 const TargetRegisterClass *ConstrainRC;
3194
3195 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
3196 // whether the branch is uniform when selecting the instruction. In
3197 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
3198 // RegBankSelect knows what it's doing if the branch condition is scc, even
3199 // though it currently does not.
3200 if (!isVCC(CondReg, *MRI)) {
3201 if (MRI->getType(CondReg) != LLT::scalar(32))
3202 return false;
3203
3204 CondPhysReg = AMDGPU::SCC;
3205 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3206 ConstrainRC = &AMDGPU::SReg_32RegClass;
3207 } else {
3208 // FIXME: Should scc->vcc copies and with exec?
3209
3210 // Unless the value of CondReg is a result of a V_CMP* instruction then we
3211 // need to insert an and with exec.
3212 if (!isVCmpResult(CondReg, *MRI)) {
3213 const bool Is64 = STI.isWave64();
3214 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3215 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3216
3217 Register TmpReg = MRI->createVirtualRegister(TRI.getBoolRC());
3218 BuildMI(*BB, &I, DL, TII.get(Opcode), TmpReg)
3219 .addReg(CondReg)
3220 .addReg(Exec)
3221 .setOperandDead(3); // Dead scc
3222 CondReg = TmpReg;
3223 }
3224
3225 CondPhysReg = TRI.getVCC();
3226 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3227 ConstrainRC = TRI.getBoolRC();
3228 }
3229
3230 if (!MRI->getRegClassOrNull(CondReg))
3231 MRI->setRegClass(CondReg, ConstrainRC);
3232
3233 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg)
3234 .addReg(CondReg);
3235 BuildMI(*BB, &I, DL, TII.get(BrOpcode))
3236 .addMBB(I.getOperand(1).getMBB());
3237
3238 I.eraseFromParent();
3239 return true;
3240}
3241
3242bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3243 MachineInstr &I) const {
3244 Register DstReg = I.getOperand(0).getReg();
3245 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
3246 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3247 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3248 if (IsVGPR)
3249 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
3250
3251 return RBI.constrainGenericRegister(
3252 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI);
3253}
3254
3255bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
3256 Register DstReg = I.getOperand(0).getReg();
3257 Register SrcReg = I.getOperand(1).getReg();
3258 Register MaskReg = I.getOperand(2).getReg();
3259 LLT Ty = MRI->getType(DstReg);
3260 LLT MaskTy = MRI->getType(MaskReg);
3261 MachineBasicBlock *BB = I.getParent();
3262 const DebugLoc &DL = I.getDebugLoc();
3263
3264 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
3265 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
3266 const RegisterBank *MaskRB = RBI.getRegBank(MaskReg, *MRI, TRI);
3267 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3268 if (DstRB != SrcRB) // Should only happen for hand written MIR.
3269 return false;
3270
3271 // Try to avoid emitting a bit operation when we only need to touch half of
3272 // the 64-bit pointer.
3273 APInt MaskOnes = VT->getKnownOnes(MaskReg).zext(64);
3274 const APInt MaskHi32 = APInt::getHighBitsSet(64, 32);
3275 const APInt MaskLo32 = APInt::getLowBitsSet(64, 32);
3276
3277 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3278 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3279
3280 if (!IsVGPR && Ty.getSizeInBits() == 64 &&
3281 !CanCopyLow32 && !CanCopyHi32) {
3282 auto MIB = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_AND_B64), DstReg)
3283 .addReg(SrcReg)
3284 .addReg(MaskReg)
3285 .setOperandDead(3); // Dead scc
3286 I.eraseFromParent();
3287 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
3288 return true;
3289 }
3290
3291 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3292 const TargetRegisterClass &RegRC
3293 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3294
3295 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, *DstRB);
3296 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, *SrcRB);
3297 const TargetRegisterClass *MaskRC =
3298 TRI.getRegClassForTypeOnBank(MaskTy, *MaskRB);
3299
3300 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
3301 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) ||
3302 !RBI.constrainGenericRegister(MaskReg, *MaskRC, *MRI))
3303 return false;
3304
3305 if (Ty.getSizeInBits() == 32) {
3306 assert(MaskTy.getSizeInBits() == 32 &&
3307 "ptrmask should have been narrowed during legalize");
3308
3309 auto NewOp = BuildMI(*BB, &I, DL, TII.get(NewOpc), DstReg)
3310 .addReg(SrcReg)
3311 .addReg(MaskReg);
3312
3313 if (!IsVGPR)
3314 NewOp.setOperandDead(3); // Dead scc
3315 I.eraseFromParent();
3316 return true;
3317 }
3318
3319 Register HiReg = MRI->createVirtualRegister(&RegRC);
3320 Register LoReg = MRI->createVirtualRegister(&RegRC);
3321
3322 // Extract the subregisters from the source pointer.
3323 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg)
3324 .addReg(SrcReg, {}, AMDGPU::sub0);
3325 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), HiReg)
3326 .addReg(SrcReg, {}, AMDGPU::sub1);
3327
3328 Register MaskedLo, MaskedHi;
3329
3330 if (CanCopyLow32) {
3331 // If all the bits in the low half are 1, we only need a copy for it.
3332 MaskedLo = LoReg;
3333 } else {
3334 // Extract the mask subregister and apply the and.
3335 Register MaskLo = MRI->createVirtualRegister(&RegRC);
3336 MaskedLo = MRI->createVirtualRegister(&RegRC);
3337
3338 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskLo)
3339 .addReg(MaskReg, {}, AMDGPU::sub0);
3340 BuildMI(*BB, &I, DL, TII.get(NewOpc), MaskedLo)
3341 .addReg(LoReg)
3342 .addReg(MaskLo);
3343 }
3344
3345 if (CanCopyHi32) {
3346 // If all the bits in the high half are 1, we only need a copy for it.
3347 MaskedHi = HiReg;
3348 } else {
3349 Register MaskHi = MRI->createVirtualRegister(&RegRC);
3350 MaskedHi = MRI->createVirtualRegister(&RegRC);
3351
3352 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskHi)
3353 .addReg(MaskReg, {}, AMDGPU::sub1);
3354 BuildMI(*BB, &I, DL, TII.get(NewOpc), MaskedHi)
3355 .addReg(HiReg)
3356 .addReg(MaskHi);
3357 }
3358
3359 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
3360 .addReg(MaskedLo)
3361 .addImm(AMDGPU::sub0)
3362 .addReg(MaskedHi)
3363 .addImm(AMDGPU::sub1);
3364 I.eraseFromParent();
3365 return true;
3366}
3367
3368/// Return the register to use for the index value, and the subregister to use
3369/// for the indirectly accessed register.
3370static std::pair<Register, unsigned>
3372 const TargetRegisterClass *SuperRC, Register IdxReg,
3373 unsigned EltSize, GISelValueTracking &ValueTracking) {
3374 Register IdxBaseReg;
3375 int Offset;
3376
3377 std::tie(IdxBaseReg, Offset) =
3378 AMDGPU::getBaseWithConstantOffset(MRI, IdxReg, &ValueTracking);
3379 if (IdxBaseReg == AMDGPU::NoRegister) {
3380 // This will happen if the index is a known constant. This should ordinarily
3381 // be legalized out, but handle it as a register just in case.
3382 assert(Offset == 0);
3383 IdxBaseReg = IdxReg;
3384 }
3385
3386 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SuperRC, EltSize);
3387
3388 // Skip out of bounds offsets, or else we would end up using an undefined
3389 // register.
3390 if (static_cast<unsigned>(Offset) >= SubRegs.size())
3391 return std::pair(IdxReg, SubRegs[0]);
3392 return std::pair(IdxBaseReg, SubRegs[Offset]);
3393}
3394
3395bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3396 MachineInstr &MI) const {
3397 Register DstReg = MI.getOperand(0).getReg();
3398 Register SrcReg = MI.getOperand(1).getReg();
3399 Register IdxReg = MI.getOperand(2).getReg();
3400
3401 LLT DstTy = MRI->getType(DstReg);
3402 LLT SrcTy = MRI->getType(SrcReg);
3403
3404 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
3405 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
3406 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI);
3407
3408 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3409 // into a waterfall loop.
3410 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3411 return false;
3412
3413 const TargetRegisterClass *SrcRC =
3414 TRI.getRegClassForTypeOnBank(SrcTy, *SrcRB);
3415 const TargetRegisterClass *DstRC =
3416 TRI.getRegClassForTypeOnBank(DstTy, *DstRB);
3417 if (!SrcRC || !DstRC)
3418 return false;
3419 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) ||
3420 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
3421 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI))
3422 return false;
3423
3424 MachineBasicBlock *BB = MI.getParent();
3425 const DebugLoc &DL = MI.getDebugLoc();
3426 const bool Is64 = DstTy.getSizeInBits() == 64;
3427
3428 unsigned SubReg;
3429 std::tie(IdxReg, SubReg) = computeIndirectRegIndex(
3430 *MRI, TRI, SrcRC, IdxReg,
3431 static_cast<unsigned>(DstTy.getSizeInBits() / 8), *VT);
3432
3433 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
3434 if (DstTy.getSizeInBits() != 32 && !Is64)
3435 return false;
3436
3437 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3438 .addReg(IdxReg);
3439
3440 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3441 BuildMI(*BB, &MI, DL, TII.get(Opc), DstReg)
3442 .addReg(SrcReg, {}, SubReg)
3443 .addReg(SrcReg, RegState::Implicit);
3444 MI.eraseFromParent();
3445 return true;
3446 }
3447
3448 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32)
3449 return false;
3450
3451 if (!STI.useVGPRIndexMode()) {
3452 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3453 .addReg(IdxReg);
3454 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg)
3455 .addReg(SrcReg, {}, SubReg)
3456 .addReg(SrcReg, RegState::Implicit);
3457 MI.eraseFromParent();
3458 return true;
3459 }
3460
3461 const MCInstrDesc &GPRIDXDesc = TII.getIndirectGPRIDXPseudo(
3462 static_cast<unsigned>(TRI.getRegSizeInBits(*SrcRC)), true);
3463 BuildMI(*BB, MI, DL, GPRIDXDesc, DstReg)
3464 .addReg(SrcReg)
3465 .addReg(IdxReg)
3466 .addImm(SubReg);
3467
3468 MI.eraseFromParent();
3469 return true;
3470}
3471
3472// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
3473bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3474 MachineInstr &MI) const {
3475 Register DstReg = MI.getOperand(0).getReg();
3476 Register VecReg = MI.getOperand(1).getReg();
3477 Register ValReg = MI.getOperand(2).getReg();
3478 Register IdxReg = MI.getOperand(3).getReg();
3479
3480 LLT VecTy = MRI->getType(DstReg);
3481 LLT ValTy = MRI->getType(ValReg);
3482 unsigned VecSize = static_cast<unsigned>(VecTy.getSizeInBits());
3483 unsigned ValSize = static_cast<unsigned>(ValTy.getSizeInBits());
3484
3485 const RegisterBank *VecRB = RBI.getRegBank(VecReg, *MRI, TRI);
3486 const RegisterBank *ValRB = RBI.getRegBank(ValReg, *MRI, TRI);
3487 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI);
3488
3489 assert(VecTy.getElementType() == ValTy);
3490
3491 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3492 // into a waterfall loop.
3493 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3494 return false;
3495
3496 const TargetRegisterClass *VecRC =
3497 TRI.getRegClassForTypeOnBank(VecTy, *VecRB);
3498 const TargetRegisterClass *ValRC =
3499 TRI.getRegClassForTypeOnBank(ValTy, *ValRB);
3500
3501 if (!RBI.constrainGenericRegister(VecReg, *VecRC, *MRI) ||
3502 !RBI.constrainGenericRegister(DstReg, *VecRC, *MRI) ||
3503 !RBI.constrainGenericRegister(ValReg, *ValRC, *MRI) ||
3504 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI))
3505 return false;
3506
3507 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3508 return false;
3509
3510 unsigned SubReg;
3511 std::tie(IdxReg, SubReg) =
3512 computeIndirectRegIndex(*MRI, TRI, VecRC, IdxReg, ValSize / 8, *VT);
3513
3514 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3515 STI.useVGPRIndexMode();
3516
3517 MachineBasicBlock *BB = MI.getParent();
3518 const DebugLoc &DL = MI.getDebugLoc();
3519
3520 if (!IndexMode) {
3521 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3522 .addReg(IdxReg);
3523
3524 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3525 VecSize, ValSize, VecRB->getID() == AMDGPU::SGPRRegBankID);
3526 BuildMI(*BB, MI, DL, RegWriteOp, DstReg)
3527 .addReg(VecReg)
3528 .addReg(ValReg)
3529 .addImm(SubReg);
3530 MI.eraseFromParent();
3531 return true;
3532 }
3533
3534 const MCInstrDesc &GPRIDXDesc = TII.getIndirectGPRIDXPseudo(
3535 static_cast<unsigned>(TRI.getRegSizeInBits(*VecRC)), false);
3536 BuildMI(*BB, MI, DL, GPRIDXDesc, DstReg)
3537 .addReg(VecReg)
3538 .addReg(ValReg)
3539 .addReg(IdxReg)
3540 .addImm(SubReg);
3541
3542 MI.eraseFromParent();
3543 return true;
3544}
3545
3546static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
3547 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
3549 return BufInfo->IsAsync;
3550 switch (Intr) {
3551 case Intrinsic::amdgcn_load_async_to_lds:
3552 case Intrinsic::amdgcn_global_load_async_lds:
3553 return true;
3554 }
3555 return false;
3556}
3557
3558bool AMDGPUInstructionSelector::selectBufferLoadLds(
3559 MachineInstr &MI, const AMDGPU::BufferIntrinsicInfo &Info) const {
3560 if (!Subtarget->hasVMemToLDSLoad())
3561 return false;
3562 auto GetArg = [&](unsigned Arg) -> MachineOperand & {
3563 return MI.getOperand(AMDGPU::getMIROperandIdx(MI, Arg));
3564 };
3565 unsigned Opc;
3566 unsigned Size = static_cast<unsigned>(GetArg(*Info.getSizeArg()).getImm());
3567
3568 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
3569 const bool HasVIndex = VIndexArg.has_value();
3570 Register VIndex;
3571 if (HasVIndex)
3572 VIndex = GetArg(*VIndexArg).getReg();
3573
3574 Register VOffset = GetArg(*Info.getVOffsetArg()).getReg();
3575 std::optional<ValueAndVReg> MaybeVOffset =
3577 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3578
3579 switch (Size) {
3580 default:
3581 return false;
3582 case 1:
3583 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3584 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3585 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3586 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3587 break;
3588 case 2:
3589 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3590 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3591 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3592 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3593 break;
3594 case 4:
3595 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3596 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3597 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3598 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3599 break;
3600 case 12:
3601 if (!Subtarget->hasLDSLoadB96_B128())
3602 return false;
3603
3604 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3605 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3606 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3607 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3608 break;
3609 case 16:
3610 if (!Subtarget->hasLDSLoadB96_B128())
3611 return false;
3612
3613 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3614 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3615 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3616 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3617 break;
3618 }
3619
3620 MachineBasicBlock *MBB = MI.getParent();
3621 const DebugLoc &DL = MI.getDebugLoc();
3622 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3623 .add(GetArg(*Info.getLDSPtrArg()));
3624
3625 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc));
3626
3627 if (HasVIndex && HasVOffset) {
3628 Register IdxReg = MRI->createVirtualRegister(TRI.getVGPR64Class());
3629 BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg)
3630 .addReg(VIndex)
3631 .addImm(AMDGPU::sub0)
3632 .addReg(VOffset)
3633 .addImm(AMDGPU::sub1);
3634
3635 MIB.addReg(IdxReg);
3636 } else if (HasVIndex) {
3637 MIB.addReg(VIndex);
3638 } else if (HasVOffset) {
3639 MIB.addReg(VOffset);
3640 }
3641
3642 MIB.add(GetArg(Info.RsrcArg));
3643 MIB.add(GetArg(*Info.getSOffsetArg()));
3644 MIB.add(GetArg(*Info.getImmOffsetArg()));
3645 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
3646 int64_t Aux = GetArg(*Info.getAuxArg()).getImm();
3647 MIB.addImm(Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL
3648 : AMDGPU::CPol::ALL_pregfx12)); // cpol
3649 MIB.addImm(
3650 Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
3651 ? 1
3652 : 0); // swz
3653 MIB.addImm(Info.IsAsync);
3654
3655 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3656 // Don't set the offset value here because the pointer points to the base of
3657 // the buffer.
3658 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3659
3660 MachinePointerInfo StorePtrI = LoadPtrI;
3661 LoadPtrI.V = PoisonValue::get(PointerType::get(MF->getFunction().getContext(),
3665
3666 auto F = LoadMMO->getFlags() &
3668 LoadMMO = MF->getMachineMemOperand(LoadPtrI, F | MachineMemOperand::MOLoad,
3669 Size, LoadMMO->getBaseAlign());
3670
3671 MachineMemOperand *StoreMMO =
3672 MF->getMachineMemOperand(StorePtrI, F | MachineMemOperand::MOStore,
3673 sizeof(int32_t), LoadMMO->getBaseAlign());
3674
3675 MIB.setMemRefs({LoadMMO, StoreMMO});
3676
3677 MI.eraseFromParent();
3678 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
3679 return true;
3680}
3681
3682/// Match a zero extend from a 32-bit value to 64-bits.
3683Register AMDGPUInstructionSelector::matchZeroExtendFromS32(Register Reg) const {
3684 Register ZExtSrc;
3685 if (mi_match(Reg, *MRI, m_GZExt(m_Reg(ZExtSrc))))
3686 return MRI->getType(ZExtSrc) == LLT::scalar(32) ? ZExtSrc : Register();
3687
3688 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3689 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI);
3690 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3691 return Register();
3692
3693 assert(Def->getNumOperands() == 3 &&
3694 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3695 if (mi_match(Def->getOperand(2).getReg(), *MRI, m_ZeroInt())) {
3696 return Def->getOperand(1).getReg();
3697 }
3698
3699 return Register();
3700}
3701
3702/// Match a sign extend from a 32-bit value to 64-bits.
3703Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
3704 Register SExtSrc;
3705 if (mi_match(Reg, *MRI, m_GSExt(m_Reg(SExtSrc))))
3706 return MRI->getType(SExtSrc) == LLT::scalar(32) ? SExtSrc : Register();
3707
3708 // Match legalized form %sext = G_MERGE_VALUES (s32 %x), G_ASHR((S32 %x, 31))
3709 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI);
3710 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3711 return Register();
3712
3713 assert(Def->getNumOperands() == 3 &&
3714 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3715 if (mi_match(Def->getOperand(2).getReg(), *MRI,
3716 m_GAShr(m_SpecificReg(Def->getOperand(1).getReg()),
3717 m_SpecificICst(31))))
3718 return Def->getOperand(1).getReg();
3719
3720 Register ZextSrc = matchZeroExtendFromS32(Reg);
3721 if (ZextSrc && VT->signBitIsZero(ZextSrc))
3722 return ZextSrc;
3723
3724 return Register();
3725}
3726
3727/// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
3728/// is 32-bit.
3730AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(Register Reg) const {
3731 return MRI->getType(Reg) == LLT::scalar(32) ? Reg
3732 : matchZeroExtendFromS32(Reg);
3733}
3734
3735/// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
3736/// is 32-bit.
3738AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(Register Reg) const {
3739 return MRI->getType(Reg) == LLT::scalar(32) ? Reg
3740 : matchSignExtendFromS32(Reg);
3741}
3742
3744AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
3745 bool IsSigned) const {
3746 if (IsSigned)
3747 return matchSignExtendFromS32OrS32(Reg);
3748
3749 return matchZeroExtendFromS32OrS32(Reg);
3750}
3751
3752Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
3753 Register AnyExtSrc;
3754 if (mi_match(Reg, *MRI, m_GAnyExt(m_Reg(AnyExtSrc))))
3755 return MRI->getType(AnyExtSrc) == LLT::scalar(32) ? AnyExtSrc : Register();
3756
3757 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 G_IMPLICIT_DEF)
3758 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI);
3759 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3760 return Register();
3761
3762 assert(Def->getNumOperands() == 3 &&
3763 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3764
3765 if (mi_match(Def->getOperand(2).getReg(), *MRI, m_GImplicitDef()))
3766 return Def->getOperand(1).getReg();
3767
3768 return Register();
3769}
3770
3771bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3772 if (!Subtarget->hasVMemToLDSLoad())
3773 return false;
3774
3775 unsigned Opc;
3776 unsigned Size = static_cast<unsigned>(MI.getOperand(3).getImm());
3777 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
3778
3779 switch (Size) {
3780 default:
3781 return false;
3782 case 1:
3783 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3784 break;
3785 case 2:
3786 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3787 break;
3788 case 4:
3789 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3790 break;
3791 case 12:
3792 if (!Subtarget->hasLDSLoadB96_B128())
3793 return false;
3794 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3795 break;
3796 case 16:
3797 if (!Subtarget->hasLDSLoadB96_B128())
3798 return false;
3799 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3800 break;
3801 }
3802
3803 MachineBasicBlock *MBB = MI.getParent();
3804 const DebugLoc &DL = MI.getDebugLoc();
3805 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3806 .add(MI.getOperand(2));
3807
3808 Register Addr = MI.getOperand(1).getReg();
3809 Register VOffset;
3810 // Try to split SAddr and VOffset. Global and LDS pointers share the same
3811 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
3812 if (!isSGPR(Addr)) {
3813 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
3814 if (isSGPR(AddrDef->Reg)) {
3815 Addr = AddrDef->Reg;
3816 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3817 Register SAddr =
3818 getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI);
3819 if (isSGPR(SAddr)) {
3820 Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
3821 if (Register Off = matchZeroExtendFromS32(PtrBaseOffset)) {
3822 Addr = SAddr;
3823 VOffset = Off;
3824 }
3825 }
3826 }
3827 }
3828
3829 if (isSGPR(Addr)) {
3831 if (!VOffset) {
3832 VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3833 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), VOffset)
3834 .addImm(0);
3835 }
3836 }
3837
3838 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opc))
3839 .addReg(Addr);
3840
3841 if (isSGPR(Addr))
3842 MIB.addReg(VOffset);
3843
3844 MIB.add(MI.getOperand(4)); // offset
3845
3846 unsigned Aux = static_cast<unsigned>(MI.getOperand(5).getImm());
3847 MIB.addImm(Aux & ~AMDGPU::CPol::VIRTUAL_BITS); // cpol
3848 MIB.addImm(isAsyncLDSDMA(IntrinsicID));
3849
3850 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3851 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3852 LoadPtrI.Offset = MI.getOperand(4).getImm();
3853 MachinePointerInfo StorePtrI = LoadPtrI;
3854 LoadPtrI.V = PoisonValue::get(PointerType::get(MF->getFunction().getContext(),
3858 auto F = LoadMMO->getFlags() &
3860 LoadMMO = MF->getMachineMemOperand(LoadPtrI, F | MachineMemOperand::MOLoad,
3861 Size, LoadMMO->getBaseAlign());
3862 MachineMemOperand *StoreMMO =
3863 MF->getMachineMemOperand(StorePtrI, F | MachineMemOperand::MOStore,
3864 sizeof(int32_t), Align(4));
3865
3866 MIB.setMemRefs({LoadMMO, StoreMMO});
3867
3868 MI.eraseFromParent();
3869 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
3870 return true;
3871}
3872
3873bool AMDGPUInstructionSelector::selectTensorLoadStore(MachineInstr &MI,
3874 Intrinsic::ID IID) const {
3875 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3876 unsigned Opc =
3877 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3878 int NumGroups = 4;
3879
3880 // A lamda function to check whether an operand is a vector of all 0s.
3881 const auto isAllZeros = [&](MachineOperand &Opnd) {
3882 const MachineInstr *DefMI = MRI->getVRegDef(Opnd.getReg());
3883 if (!DefMI)
3884 return false;
3885 return llvm::isBuildVectorAllZeros(*DefMI, *MRI, true);
3886 };
3887
3888 // Use _D2 version if both group 2 and 3 are zero-initialized.
3889 if (isAllZeros(MI.getOperand(3)) && isAllZeros(MI.getOperand(4))) {
3890 NumGroups = 2;
3891 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3892 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3893 }
3894
3895 // TODO: Handle the fifth group: MI.getOpetand(5), which is silently ignored
3896 // for now because all existing targets only support up to 4 groups.
3897 MachineBasicBlock *MBB = MI.getParent();
3898 auto MIB = BuildMI(*MBB, &MI, MI.getDebugLoc(), TII.get(Opc))
3899 .add(MI.getOperand(1)) // D# group 0
3900 .add(MI.getOperand(2)); // D# group 1
3901
3902 if (NumGroups >= 4) { // Has at least 4 groups
3903 MIB.add(MI.getOperand(3)) // D# group 2
3904 .add(MI.getOperand(4)); // D# group 3
3905 }
3906
3907 MIB.addImm(0) // r128
3908 .add(MI.getOperand(6)); // cpol
3909
3910 MI.eraseFromParent();
3911 return true;
3912}
3913
3914bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3915 MachineInstr &MI) const {
3916 unsigned OpcodeOpIdx =
3917 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3918 MI.setDesc(
3919 TII.get(static_cast<unsigned>(MI.getOperand(OpcodeOpIdx).getImm())));
3920 MI.removeOperand(OpcodeOpIdx);
3921 MI.addImplicitDefUseOperands(*MI.getMF());
3922 constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
3923 return true;
3924}
3925
3926// FIXME: This should be removed and let the patterns select. We just need the
3927// AGPR/VGPR combination versions.
3928bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3929 unsigned Opc;
3930 switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
3931 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3932 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3933 break;
3934 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3935 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3936 break;
3937 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3938 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3939 break;
3940 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3941 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3942 break;
3943 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3944 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3945 break;
3946 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3947 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3948 break;
3949 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3950 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3951 break;
3952 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3953 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3954 break;
3955 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3956 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3957 break;
3958 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3959 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3960 break;
3961 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3962 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3963 break;
3964 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3965 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3966 break;
3967 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3968 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3969 break;
3970 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3971 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3972 break;
3973 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3974 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3975 break;
3976 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3977 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3978 break;
3979 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3980 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3981 break;
3982 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
3983 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
3984 break;
3985 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
3986 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
3987 break;
3988 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
3989 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
3990 break;
3991 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3992 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3993 break;
3994 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
3995 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
3996 break;
3997 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
3998 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
3999 break;
4000 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
4001 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
4002 break;
4003 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
4004 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
4005 break;
4006 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
4007 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
4008 break;
4009 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
4010 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
4011 break;
4012 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
4013 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
4014 break;
4015 default:
4016 llvm_unreachable("unhandled smfmac intrinsic");
4017 }
4018
4019 auto VDst_In = MI.getOperand(4);
4020
4021 MI.setDesc(TII.get(Opc));
4022 MI.removeOperand(4); // VDst_In
4023 MI.removeOperand(1); // Intrinsic ID
4024 MI.addOperand(VDst_In); // Readd VDst_In to the end
4025 MI.addImplicitDefUseOperands(*MI.getMF());
4026 const MCInstrDesc &MCID = MI.getDesc();
4027 if (MCID.getOperandConstraint(0, MCOI::EARLY_CLOBBER) != -1) {
4028 MI.getOperand(0).setIsEarlyClobber(true);
4029 }
4030 return true;
4031}
4032
4033bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
4034 MachineInstr &MI, Intrinsic::ID IntrID) const {
4035 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
4036 !Subtarget->hasPermlane16Swap())
4037 return false;
4038 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
4039 !Subtarget->hasPermlane32Swap())
4040 return false;
4041
4042 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
4043 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
4044 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
4045
4046 MI.removeOperand(2);
4047 MI.setDesc(TII.get(Opcode));
4048 MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
4049
4050 MachineOperand &FI = MI.getOperand(4);
4052
4053 constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
4054 return true;
4055}
4056
4057bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
4058 Register DstReg = MI.getOperand(0).getReg();
4059 Register SrcReg = MI.getOperand(1).getReg();
4060 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
4061 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4062 MachineBasicBlock *MBB = MI.getParent();
4063 const DebugLoc &DL = MI.getDebugLoc();
4064
4065 if (IsVALU) {
4066 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg)
4067 .addImm(Subtarget->getWavefrontSizeLog2())
4068 .addReg(SrcReg);
4069 } else {
4070 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg)
4071 .addReg(SrcReg)
4072 .addImm(Subtarget->getWavefrontSizeLog2())
4073 .setOperandDead(3); // Dead scc
4074 }
4075
4076 const TargetRegisterClass &RC =
4077 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4078 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI))
4079 return false;
4080
4081 MI.eraseFromParent();
4082 return true;
4083}
4084
4085bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4086 MachineInstr &MI) const {
4087 assert(MI.getNumOperands() == 4);
4088 MachineBasicBlock *MBB = MI.getParent();
4089 const DebugLoc &DL = MI.getDebugLoc();
4090
4091 Register DstReg = MI.getOperand(0).getReg();
4092 Register ValReg = MI.getOperand(2).getReg();
4093 Register IdxReg = MI.getOperand(3).getReg();
4094
4095 const LLT DstTy = MRI->getType(DstReg);
4096 uint64_t DstSize = DstTy.getSizeInBits();
4097 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
4098 const TargetRegisterClass *DstRC =
4099 TRI.getRegClassForSizeOnBank(DstSize, *DstRB);
4100
4101 if (DstTy != LLT::scalar(32))
4102 return false;
4103
4104 if (!Subtarget->supportsBPermute())
4105 return false;
4106
4107 // If we can bpermute across the whole wave, then just do that
4108 if (Subtarget->supportsWaveWideBPermute()) {
4109 Register ShiftIdxReg = MRI->createVirtualRegister(DstRC);
4110 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), ShiftIdxReg)
4111 .addImm(2)
4112 .addReg(IdxReg);
4113
4114 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::DS_BPERMUTE_B32), DstReg)
4115 .addReg(ShiftIdxReg)
4116 .addReg(ValReg)
4117 .addImm(0);
4118 } else {
4119 // Otherwise, we need to make use of whole wave mode
4120 assert(Subtarget->isWave64());
4121
4122 // Set inactive lanes to poison
4123 Register UndefValReg =
4124 MRI->createVirtualRegister(TRI.getRegClass(AMDGPU::SReg_32RegClassID));
4125 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefValReg);
4126
4127 Register UndefExecReg = MRI->createVirtualRegister(
4128 TRI.getRegClass(AMDGPU::SReg_64_XEXECRegClassID));
4129 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefExecReg);
4130
4131 Register PoisonValReg = MRI->createVirtualRegister(DstRC);
4132 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SET_INACTIVE_B32), PoisonValReg)
4133 .addImm(0)
4134 .addReg(ValReg)
4135 .addImm(0)
4136 .addReg(UndefValReg)
4137 .addReg(UndefExecReg);
4138
4139 // ds_bpermute requires index to be multiplied by 4
4140 Register ShiftIdxReg = MRI->createVirtualRegister(DstRC);
4141 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), ShiftIdxReg)
4142 .addImm(2)
4143 .addReg(IdxReg);
4144
4145 Register PoisonIdxReg = MRI->createVirtualRegister(DstRC);
4146 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SET_INACTIVE_B32), PoisonIdxReg)
4147 .addImm(0)
4148 .addReg(ShiftIdxReg)
4149 .addImm(0)
4150 .addReg(UndefValReg)
4151 .addReg(UndefExecReg);
4152
4153 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(DstRC);
4154 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SET_INACTIVE_B32),
4155 PoisonUnshiftedIdxReg)
4156 .addImm(0)
4157 .addReg(IdxReg)
4158 .addImm(0)
4159 .addReg(UndefValReg)
4160 .addReg(UndefExecReg);
4161
4162 // Get permutation of each half, then we'll select which one to use
4163 Register SameSidePermReg = MRI->createVirtualRegister(DstRC);
4164 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::DS_BPERMUTE_B32), SameSidePermReg)
4165 .addReg(PoisonIdxReg)
4166 .addReg(PoisonValReg)
4167 .addImm(0);
4168
4169 Register SwappedValReg = MRI->createVirtualRegister(DstRC);
4170 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_PERMLANE64_B32), SwappedValReg)
4171 .addReg(PoisonValReg);
4172
4173 Register OppSidePermReg = MRI->createVirtualRegister(DstRC);
4174 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::DS_BPERMUTE_B32), OppSidePermReg)
4175 .addReg(PoisonIdxReg)
4176 .addReg(SwappedValReg)
4177 .addImm(0);
4178
4179 Register WWMSwapPermReg = MRI->createVirtualRegister(DstRC);
4180 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::STRICT_WWM), WWMSwapPermReg)
4181 .addReg(OppSidePermReg);
4182
4183 // Select which side to take the permute from
4184 // We can get away with only using mbcnt_lo here since we're only
4185 // trying to detect which side of 32 each lane is on, and mbcnt_lo
4186 // returns 32 for lanes 32-63.
4187 Register ThreadIDReg = MRI->createVirtualRegister(DstRC);
4188 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_MBCNT_LO_U32_B32_e64), ThreadIDReg)
4189 .addImm(-1)
4190 .addImm(0);
4191
4192 Register XORReg = MRI->createVirtualRegister(DstRC);
4193 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_XOR_B32_e64), XORReg)
4194 .addReg(ThreadIDReg)
4195 .addReg(PoisonUnshiftedIdxReg);
4196
4197 Register ANDReg = MRI->createVirtualRegister(DstRC);
4198 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_AND_B32_e64), ANDReg)
4199 .addReg(XORReg)
4200 .addImm(32);
4201
4202 Register CompareReg = MRI->createVirtualRegister(
4203 TRI.getRegClass(AMDGPU::SReg_64_XEXECRegClassID));
4204 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), CompareReg)
4205 .addReg(ANDReg)
4206 .addImm(0);
4207
4208 // Finally do the selection
4209 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
4210 .addImm(0)
4211 .addReg(WWMSwapPermReg)
4212 .addImm(0)
4213 .addReg(SameSidePermReg)
4214 .addReg(CompareReg);
4215 }
4216
4217 MI.eraseFromParent();
4218 return true;
4219}
4220
4221// Match BITOP3 operation and return a number of matched instructions plus
4222// truth table.
4223static std::pair<unsigned, uint8_t> BitOp3_Op(Register R,
4225 const MachineRegisterInfo &MRI) {
4226 unsigned NumOpcodes = 0;
4227 uint8_t LHSBits, RHSBits;
4228
4229 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool {
4230 // Define truth table given Src0, Src1, Src2 bits permutations:
4231 // 0 0 0
4232 // 0 0 1
4233 // 0 1 0
4234 // 0 1 1
4235 // 1 0 0
4236 // 1 0 1
4237 // 1 1 0
4238 // 1 1 1
4239 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4240
4241 if (mi_match(Op, MRI, m_AllOnesInt())) {
4242 Bits = 0xff;
4243 return true;
4244 }
4245 if (mi_match(Op, MRI, m_ZeroInt())) {
4246 Bits = 0;
4247 return true;
4248 }
4249
4250 for (unsigned I = 0; I < Src.size(); ++I) {
4251 // Try to find existing reused operand
4252 if (Src[I] == Op) {
4253 Bits = SrcBits[I];
4254 return true;
4255 }
4256 // Try to replace parent operator
4257 if (Src[I] == R) {
4258 Bits = SrcBits[I];
4259 Src[I] = Op;
4260 return true;
4261 }
4262 }
4263
4264 if (Src.size() == 3) {
4265 // No room left for operands. Try one last time, there can be a 'not' of
4266 // one of our source operands. In this case we can compute the bits
4267 // without growing Src vector.
4268 Register LHS;
4269 if (mi_match(Op, MRI, m_Not(m_Reg(LHS)))) {
4271 for (unsigned I = 0; I < Src.size(); ++I) {
4272 if (Src[I] == LHS) {
4273 Bits = ~SrcBits[I];
4274 return true;
4275 }
4276 }
4277 }
4278
4279 return false;
4280 }
4281
4282 Bits = SrcBits[Src.size()];
4283 Src.push_back(Op);
4284 return true;
4285 };
4286
4287 MachineInstr *MI = MRI.getVRegDef(R);
4288 switch (MI->getOpcode()) {
4289 case TargetOpcode::G_AND:
4290 case TargetOpcode::G_OR:
4291 case TargetOpcode::G_XOR: {
4292 Register LHS = getSrcRegIgnoringCopies(MI->getOperand(1).getReg(), MRI);
4293 Register RHS = getSrcRegIgnoringCopies(MI->getOperand(2).getReg(), MRI);
4294
4295 SmallVector<Register, 3> Backup(Src.begin(), Src.end());
4296 if (!getOperandBits(LHS, LHSBits) ||
4297 !getOperandBits(RHS, RHSBits)) {
4298 Src = std::move(Backup);
4299 return {};
4300 }
4301
4302 // Recursion is naturally limited by the size of the operand vector.
4303 //
4304 // When LHS and RHS share a common sub-expression, one side's recursion
4305 // may decompose that sub-expression and replace the Src slot the other
4306 // side occupies with sub-operands via the "replace parent" path in
4307 // getOperandBits. The other side's cached bit-pattern then refers to a
4308 // slot whose contents changed, producing a wrong truth table.
4309 //
4310 // We detect this in three ways:
4311 // (A) If LHS recursed, its truth table is valid against the Src state
4312 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4313 // then mutates a Src slot that LHSBits depends on, LHSBits is
4314 // stale.
4315 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4316 // refers to a specific Src slot. If that slot's contents changed
4317 // (by either recursion), RHSBits is stale.
4318 // (C) Symmetrically for LHS if it did not recurse.
4319 SmallVector<Register, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4320 uint8_t LHSBitsOrig = LHSBits;
4321 uint8_t RHSBitsOrig = RHSBits;
4322
4323 auto LHSOp = BitOp3_Op(LHS, Src, MRI);
4324 if (LHSOp.first) {
4325 NumOpcodes += LHSOp.first;
4326 LHSBits = LHSOp.second;
4327 }
4328
4329 SmallVector<Register, 3> SrcAfterLHS(Src.begin(), Src.end());
4330
4331 auto RHSOp = BitOp3_Op(RHS, Src, MRI);
4332 if (RHSOp.first) {
4333 NumOpcodes += RHSOp.first;
4334 RHSBits = RHSOp.second;
4335 }
4336
4337 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4338 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4339 if (Slot < 0 || Slot > 2)
4340 return false;
4341 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4342 const int Shifts[3] = {4, 2, 1};
4343 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4344 };
4345
4346 // findSlot: locate the Src slot a getOperandBits result depends on,
4347 // including negated (NOT) patterns that getOperandBits resolves via
4348 // the ~SrcBits[I] shortcut.
4349 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4350 auto findSlot = [&](uint8_t Bits, Register Op,
4351 const SmallVectorImpl<Register> &S) -> int {
4352 Register NegatedInner;
4353 bool IsNegationOp = mi_match(Op, MRI, m_Not(m_Reg(NegatedInner)));
4354 if (IsNegationOp)
4355 NegatedInner = getSrcRegIgnoringCopies(NegatedInner, MRI);
4356 for (int I = 0; I < (int)S.size(); I++) {
4357 if (Bits == SrcBitsConst[I] && S[I] == Op)
4358 return I;
4359 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4360 S[I] == NegatedInner)
4361 return I;
4362 }
4363 return -1;
4364 };
4365
4366 bool Stale = false;
4367
4368 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4369 // Check if RHS recursion mutated a slot that LHSBits uses.
4370 if (LHSOp.first) {
4371 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4372 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4373 dependsOnSlot(LHSBits, I)) {
4374 Stale = true;
4375 break;
4376 }
4377 }
4378 }
4379
4380 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4381 // SrcBeforeRecurse. Check if that slot was mutated since then.
4382 if (!Stale && !RHSOp.first) {
4383 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4384 if (Slot >= 0 &&
4385 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4386 Stale = true;
4387 }
4388
4389 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4390 // SrcBeforeRecurse. Check if that slot was mutated since then.
4391 if (!Stale && !LHSOp.first) {
4392 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4393 if (Slot >= 0 &&
4394 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4395 Stale = true;
4396 }
4397
4398 if (Stale) {
4399 Src = std::move(SrcBeforeRecurse);
4400 LHSBits = LHSBitsOrig;
4401 RHSBits = RHSBitsOrig;
4402 NumOpcodes = 0;
4403 }
4404 break;
4405 }
4406 default:
4407 return {};
4408 }
4409
4410 uint8_t TTbl;
4411 switch (MI->getOpcode()) {
4412 case TargetOpcode::G_AND:
4413 TTbl = LHSBits & RHSBits;
4414 break;
4415 case TargetOpcode::G_OR:
4416 TTbl = LHSBits | RHSBits;
4417 break;
4418 case TargetOpcode::G_XOR:
4419 TTbl = LHSBits ^ RHSBits;
4420 break;
4421 default:
4422 break;
4423 }
4424
4425 return std::make_pair(NumOpcodes + 1, TTbl);
4426}
4427
4428bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
4429 if (!Subtarget->hasBitOp3Insts())
4430 return false;
4431
4432 Register DstReg = MI.getOperand(0).getReg();
4433 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
4434 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4435 if (!IsVALU)
4436 return false;
4437
4439 uint8_t TTbl;
4440 unsigned NumOpcodes;
4441
4442 std::tie(NumOpcodes, TTbl) = BitOp3_Op(DstReg, Src, *MRI);
4443
4444 // Src.empty() case can happen if all operands are all zero or all ones.
4445 // Normally it shall be optimized out before reaching this.
4446 if (NumOpcodes < 2 || Src.empty())
4447 return false;
4448
4449 // RegBankSelect splits wider VALU logic ops and widens 1-bit ones, so only
4450 // 16 and 32 bit types reach here. Note that <2 x i16> is 32 bits wide.
4451 unsigned Size = MRI->getType(DstReg).getSizeInBits();
4452 assert((Size == 16 || Size == 32) && "unexpected VALU logic op size");
4453 const bool IsB32 = Size == 32;
4454 if (NumOpcodes == 2 && IsB32) {
4455 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4456 // asm more readable. This cannot be modeled with AddedComplexity because
4457 // selector does not know how many operations did we match.
4458 if (mi_match(MI, *MRI, m_GXor(m_GXor(m_Reg(), m_Reg()), m_Reg())) ||
4459 mi_match(MI, *MRI, m_GOr(m_GOr(m_Reg(), m_Reg()), m_Reg())) ||
4460 mi_match(MI, *MRI, m_GOr(m_GAnd(m_Reg(), m_Reg()), m_Reg())))
4461 return false;
4462 } else if (NumOpcodes < 4) {
4463 // For a uniform case threshold should be higher to account for moves
4464 // between VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be
4465 // in SGPRs and a readtfirstlane after.
4466 return false;
4467 }
4468
4469 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4470 if (!IsB32 && STI.hasTrue16BitInsts())
4471 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4472 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4473 unsigned CBL = STI.getConstantBusLimit(Opc);
4474 MachineBasicBlock *MBB = MI.getParent();
4475 const DebugLoc &DL = MI.getDebugLoc();
4476
4477 for (unsigned I = 0; I < Src.size(); ++I) {
4478 const RegisterBank *RB = RBI.getRegBank(Src[I], *MRI, TRI);
4479 if (RB->getID() != AMDGPU::SGPRRegBankID)
4480 continue;
4481 if (CBL > 0) {
4482 --CBL;
4483 continue;
4484 }
4485 Register NewReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4486 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::COPY), NewReg)
4487 .addReg(Src[I]);
4488 Src[I] = NewReg;
4489 }
4490
4491 // Last operand can be ignored, turning a ternary operation into a binary.
4492 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4493 // 'c' with 'a' here without changing the answer. In some pathological
4494 // cases it should be possible to get an operation with a single operand
4495 // too if optimizer would not catch it.
4496 while (Src.size() < 3)
4497 Src.push_back(Src[0]);
4498
4499 auto MIB = BuildMI(*MBB, MI, DL, TII.get(Opc), DstReg);
4500 if (!IsB32)
4501 MIB.addImm(0); // src_mod0
4502 MIB.addReg(Src[0]);
4503 if (!IsB32)
4504 MIB.addImm(0); // src_mod1
4505 MIB.addReg(Src[1]);
4506 if (!IsB32)
4507 MIB.addImm(0); // src_mod2
4508 MIB.addReg(Src[2])
4509 .addImm(TTbl);
4510 if (!IsB32)
4511 MIB.addImm(0); // op_sel
4512
4513 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
4514 MI.eraseFromParent();
4515
4516 return true;
4517}
4518
4519bool AMDGPUInstructionSelector::selectWriteRegister(MachineInstr &MI) const {
4520 const MDString *RegStr =
4521 cast<MDString>(MI.getOperand(0).getMetadata()->getOperand(0));
4522 Register SrcReg = MI.getOperand(1).getReg();
4523 LLT Ty = MRI->getType(SrcReg);
4524
4525 Register PhysReg = Subtarget->getTargetLowering()->getRegisterByName(
4526 RegStr->getString().data(), Ty, *MF);
4527 if (!PhysReg) {
4528 const Function &Fn = MF->getFunction();
4529 Fn.getContext().diagnose(DiagnosticInfoGenericWithLoc(
4530 "invalid register \"" + Twine(RegStr->getString()) +
4531 "\" for llvm.write_register",
4532 Fn, MI.getDebugLoc()));
4533 MI.eraseFromParent();
4534 return true;
4535 }
4536
4537 if (!RBI.constrainGenericRegister(
4538 SrcReg, *TRI.getSGPRClassForBitWidth(Ty.getSizeInBits()), *MRI))
4539 return false;
4540
4541 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(AMDGPU::COPY), PhysReg)
4542 .addReg(SrcReg);
4543 MI.eraseFromParent();
4544 return true;
4545}
4546
4547bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
4548 Register SrcReg = MI.getOperand(0).getReg();
4549 if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, *MRI))
4550 return false;
4551
4552 MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
4553 Register SP =
4554 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4555 Register WaveAddr = getWaveAddress(DefMI);
4556 MachineBasicBlock *MBB = MI.getParent();
4557 const DebugLoc &DL = MI.getDebugLoc();
4558
4559 if (!WaveAddr) {
4560 WaveAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
4561 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHL_B32), WaveAddr)
4562 .addReg(SrcReg)
4563 .addImm(Subtarget->getWavefrontSizeLog2())
4564 .setOperandDead(3); // Dead scc
4565 }
4566
4567 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), SP)
4568 .addReg(WaveAddr);
4569
4570 MI.eraseFromParent();
4571 return true;
4572}
4573
4575
4576 if (!I.isPreISelOpcode()) {
4577 if (I.isCopy())
4578 return selectCOPY(I);
4579 return true;
4580 }
4581
4582 switch (I.getOpcode()) {
4583 case TargetOpcode::G_AND:
4584 case TargetOpcode::G_OR:
4585 case TargetOpcode::G_XOR:
4586 if (selectBITOP3(I))
4587 return true;
4588 if (selectImpl(I, *CoverageInfo))
4589 return true;
4590 return selectG_AND_OR_XOR(I);
4591 case TargetOpcode::G_ADD:
4592 case TargetOpcode::G_SUB:
4593 case TargetOpcode::G_PTR_ADD:
4594 if (selectImpl(I, *CoverageInfo))
4595 return true;
4596 return selectG_ADD_SUB(I);
4597 case TargetOpcode::G_UADDO:
4598 case TargetOpcode::G_USUBO:
4599 case TargetOpcode::G_UADDE:
4600 case TargetOpcode::G_USUBE:
4601 return selectG_UADDO_USUBO_UADDE_USUBE(I);
4602 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4603 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4604 return selectG_AMDGPU_MAD_64_32(I);
4605 case TargetOpcode::G_INTTOPTR:
4606 case TargetOpcode::G_BITCAST:
4607 case TargetOpcode::G_PTRTOINT:
4608 case TargetOpcode::G_FREEZE:
4609 return selectCOPY(I);
4610 case TargetOpcode::G_FNEG:
4611 if (selectImpl(I, *CoverageInfo))
4612 return true;
4613 return selectG_FNEG(I);
4614 case TargetOpcode::G_FABS:
4615 if (selectImpl(I, *CoverageInfo))
4616 return true;
4617 return selectG_FABS(I);
4618 case TargetOpcode::G_EXTRACT:
4619 return selectG_EXTRACT(I);
4620 case TargetOpcode::G_MERGE_VALUES:
4621 case TargetOpcode::G_CONCAT_VECTORS:
4622 return selectG_MERGE_VALUES(I);
4623 case TargetOpcode::G_UNMERGE_VALUES:
4624 return selectG_UNMERGE_VALUES(I);
4625 case TargetOpcode::G_BUILD_VECTOR:
4626 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4627 return selectG_BUILD_VECTOR(I);
4628 case TargetOpcode::G_IMPLICIT_DEF:
4629 return selectG_IMPLICIT_DEF(I);
4630 case TargetOpcode::G_INSERT:
4631 return selectG_INSERT(I);
4632 case TargetOpcode::G_INTRINSIC:
4633 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4634 return selectG_INTRINSIC(I);
4635 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4636 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4637 return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
4638 case TargetOpcode::G_ICMP:
4639 case TargetOpcode::G_FCMP:
4640 if (selectG_ICMP_or_FCMP(I))
4641 return true;
4642 return selectImpl(I, *CoverageInfo);
4643 case TargetOpcode::G_LOAD:
4644 case TargetOpcode::G_ZEXTLOAD:
4645 case TargetOpcode::G_SEXTLOAD:
4646 case TargetOpcode::G_STORE:
4647 case TargetOpcode::G_ATOMIC_CMPXCHG:
4648 case TargetOpcode::G_ATOMICRMW_XCHG:
4649 case TargetOpcode::G_ATOMICRMW_ADD:
4650 case TargetOpcode::G_ATOMICRMW_SUB:
4651 case TargetOpcode::G_ATOMICRMW_AND:
4652 case TargetOpcode::G_ATOMICRMW_OR:
4653 case TargetOpcode::G_ATOMICRMW_XOR:
4654 case TargetOpcode::G_ATOMICRMW_MIN:
4655 case TargetOpcode::G_ATOMICRMW_MAX:
4656 case TargetOpcode::G_ATOMICRMW_UMIN:
4657 case TargetOpcode::G_ATOMICRMW_UMAX:
4658 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4659 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4660 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4661 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4662 case TargetOpcode::G_ATOMICRMW_FADD:
4663 case TargetOpcode::G_ATOMICRMW_FMIN:
4664 case TargetOpcode::G_ATOMICRMW_FMAX:
4665 return selectG_LOAD_STORE_ATOMICRMW(I);
4666 case TargetOpcode::G_SELECT:
4667 return selectG_SELECT(I);
4668 case TargetOpcode::G_TRUNC:
4669 return selectG_TRUNC(I);
4670 case TargetOpcode::G_SEXT:
4671 case TargetOpcode::G_ZEXT:
4672 case TargetOpcode::G_ANYEXT:
4673 case TargetOpcode::G_SEXT_INREG:
4674 // This is a workaround. For extension from type i1, `selectImpl()` uses
4675 // patterns from TD file and generates an illegal VGPR to SGPR COPY as type
4676 // i1 can only be hold in a SGPR class.
4677 if (MRI->getType(I.getOperand(1).getReg()) != LLT::scalar(1) &&
4678 selectImpl(I, *CoverageInfo))
4679 return true;
4680 return selectG_SZA_EXT(I);
4681 case TargetOpcode::G_FPEXT:
4682 if (selectG_FPEXT(I))
4683 return true;
4684 return selectImpl(I, *CoverageInfo);
4685 case TargetOpcode::G_BRCOND:
4686 return selectG_BRCOND(I);
4687 case TargetOpcode::G_GLOBAL_VALUE:
4688 return selectG_GLOBAL_VALUE(I);
4689 case TargetOpcode::G_PTRMASK:
4690 return selectG_PTRMASK(I);
4691 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4692 return selectG_EXTRACT_VECTOR_ELT(I);
4693 case TargetOpcode::G_INSERT_VECTOR_ELT:
4694 return selectG_INSERT_VECTOR_ELT(I);
4695 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4696 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4697 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4698 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4699 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4700 const AMDGPU::ImageDimIntrinsicInfo *Intr =
4702 assert(Intr && "not an image intrinsic with image pseudo");
4703 return selectImageIntrinsic(I, Intr);
4704 }
4705 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4706 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4707 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4708 return selectBVHIntersectRayIntrinsic(I);
4709 case AMDGPU::G_SBFX:
4710 case AMDGPU::G_UBFX:
4711 return selectG_SBFX_UBFX(I);
4712 case AMDGPU::G_SI_CALL:
4713 I.setDesc(TII.get(AMDGPU::SI_CALL));
4714 return true;
4715 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4716 return selectWaveAddress(I);
4717 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4718 I.setDesc(TII.get(AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4719 return true;
4720 }
4721 case AMDGPU::G_STACKRESTORE:
4722 return selectStackRestore(I);
4723 case TargetOpcode::G_WRITE_REGISTER:
4724 return selectWriteRegister(I);
4725 case AMDGPU::G_PHI:
4726 return selectPHI(I);
4727 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4728 return selectCOPY_SCC_VCC(I);
4729 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4730 return selectCOPY_VCC_SCC(I);
4731 case AMDGPU::G_AMDGPU_READANYLANE:
4732 return selectReadAnyLane(I);
4733 case TargetOpcode::G_CONSTANT:
4734 case TargetOpcode::G_FCONSTANT:
4735 default:
4736 return selectImpl(I, *CoverageInfo);
4737 }
4738 return false;
4739}
4740
4742AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
4743 return {{
4744 [=](MachineInstrBuilder &MIB) { MIB.add(Root); }
4745 }};
4746
4747}
4748
4749std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4750 Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const {
4751 unsigned Mods = 0;
4752 MachineInstr *MI = getDefIgnoringCopies(Src, *MRI);
4753
4754 if (MI->getOpcode() == AMDGPU::G_FNEG) {
4755 Src = MI->getOperand(1).getReg();
4756 Mods |= SISrcMods::NEG;
4757 MI = getDefIgnoringCopies(Src, *MRI);
4758 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4759 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
4760 // denormal mode, but we're implicitly canonicalizing in a source operand.
4761 const ConstantFP *LHS =
4762 getConstantFPVRegVal(MI->getOperand(1).getReg(), *MRI);
4763 if (LHS && LHS->isZero()) {
4764 Mods |= SISrcMods::NEG;
4765 Src = MI->getOperand(2).getReg();
4766 }
4767 }
4768
4769 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
4770 Src = MI->getOperand(1).getReg();
4771 Mods |= SISrcMods::ABS;
4772 }
4773
4774 if (OpSel)
4775 Mods |= SISrcMods::OP_SEL_0;
4776
4777 return std::pair(Src, Mods);
4778}
4779
4780std::pair<Register, unsigned>
4781AMDGPUInstructionSelector::selectVOP3PModsF32Impl(Register Src) const {
4782 unsigned Mods;
4783 std::tie(Src, Mods) = selectVOP3ModsImpl(Src);
4784 Mods |= SISrcMods::OP_SEL_1;
4785 return std::pair(Src, Mods);
4786}
4787
4788Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4789 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
4790 bool ForceVGPR) const {
4791 if ((Mods != 0 || ForceVGPR) &&
4792 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4793
4794 // If we looked through copies to find source modifiers on an SGPR operand,
4795 // we now have an SGPR register source. To avoid potentially violating the
4796 // constant bus restriction, we need to insert a copy to a VGPR.
4797 Register VGPRSrc = MRI->cloneVirtualRegister(Root.getReg());
4798 BuildMI(*InsertPt->getParent(), InsertPt, InsertPt->getDebugLoc(),
4799 TII.get(AMDGPU::COPY), VGPRSrc)
4800 .addReg(Src);
4801 Src = VGPRSrc;
4802 }
4803
4804 return Src;
4805}
4806
4807/// Some instructions must have 32-bit sources. With real true16 instructions a
4808/// 16-bit VALU value lives in a VGPR_16, which they cannot read, so place it in
4809/// the low half of a new 32-bit VGPR.
4811AMDGPUInstructionSelector::widenSrcIfVGPR16(Register Src,
4812 MachineInstr *InsertPt) const {
4813 if (!Subtarget->useRealTrue16Insts() || MRI->getType(Src) != LLT::scalar(16))
4814 return Src;
4815
4816 const RegisterBank *SrcRB = RBI.getRegBank(Src, *MRI, TRI);
4817 if (!SrcRB || SrcRB->getID() != AMDGPU::VGPRRegBankID)
4818 return Src;
4819
4820 MachineIRBuilder B(*InsertPt);
4821
4822 Register ImpDefReg = MRI->createVirtualRegister(&AMDGPU::VGPR_16RegClass);
4823 B.buildInstr(TargetOpcode::IMPLICIT_DEF).addDef(ImpDefReg);
4824
4825 Register DstReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4826 B.buildInstr(AMDGPU::REG_SEQUENCE)
4827 .addDef(DstReg)
4828 .addReg(Src)
4829 .addImm(AMDGPU::lo16)
4830 .addReg(ImpDefReg)
4831 .addImm(AMDGPU::hi16);
4832
4833 return DstReg;
4834}
4835
4836///
4837/// This will select either an SGPR or VGPR operand and will save us from
4838/// having to write an extra tablegen pattern.
4840AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
4841 return {{
4842 [=](MachineInstrBuilder &MIB) { MIB.add(Root); }
4843 }};
4844}
4845
4847AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
4848 Register Src;
4849 unsigned Mods;
4850 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg());
4851
4852 return {{
4853 [=](MachineInstrBuilder &MIB) {
4854 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4855 },
4856 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
4857 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
4858 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
4859 }};
4860}
4861
4863AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
4864 Register Src;
4865 unsigned Mods;
4866 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(),
4867 /*IsCanonicalizing=*/true,
4868 /*AllowAbs=*/false);
4869
4870 return {{
4871 [=](MachineInstrBuilder &MIB) {
4872 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4873 },
4874 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
4875 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
4876 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
4877 }};
4878}
4879
4881AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
4882 return {{
4883 [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
4884 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp
4885 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod
4886 }};
4887}
4888
4890AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
4891 Register Src;
4892 unsigned Mods;
4893 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg());
4894
4895 return {{
4896 [=](MachineInstrBuilder &MIB) {
4897 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4898 },
4899 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
4900 }};
4901}
4902
4904AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4905 MachineOperand &Root) const {
4906 Register Src;
4907 unsigned Mods;
4908 std::tie(Src, Mods) =
4909 selectVOP3ModsImpl(Root.getReg(), /*IsCanonicalizing=*/false);
4910
4911 return {{
4912 [=](MachineInstrBuilder &MIB) {
4913 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4914 },
4915 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
4916 }};
4917}
4918
4920AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
4921 Register Src;
4922 unsigned Mods;
4923 std::tie(Src, Mods) =
4924 selectVOP3ModsImpl(Root.getReg(), /*IsCanonicalizing=*/true,
4925 /*AllowAbs=*/false);
4926
4927 return {{
4928 [=](MachineInstrBuilder &MIB) {
4929 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4930 },
4931 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
4932 }};
4933}
4934
4936AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
4937 Register Reg = Root.getReg();
4938 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI);
4939 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS)
4940 return {};
4941 return {{
4942 [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
4943 }};
4944}
4945
4947AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const {
4949 if (isExtractHiElt(*MRI, Root.getReg(), Unused))
4950 return {};
4951 return selectVOP3NoMods(Root);
4952}
4953
4954enum class SrcStatus {
4959 // This means current op = [op_upper, op_lower] and src = -op_lower.
4962 // This means current op = [op_upper, op_lower] and src = [op_upper,
4963 // -op_lower].
4971};
4972/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
4973static bool isTruncHalf(const MachineInstr *MI,
4974 const MachineRegisterInfo &MRI) {
4975 if (MI->getOpcode() != AMDGPU::G_TRUNC)
4976 return false;
4977
4978 uint64_t DstSize = MRI.getType(MI->getOperand(0).getReg()).getSizeInBits();
4979 uint64_t SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
4980 return DstSize * 2 == SrcSize;
4981}
4982
4983/// Test if the MI is logic shift right with half bits,
4984/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
4985static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4986 if (MI->getOpcode() != AMDGPU::G_LSHR)
4987 return false;
4988
4989 Register ShiftSrc;
4990 std::optional<ValueAndVReg> ShiftAmt;
4991 if (mi_match(MI->getOperand(0).getReg(), MRI,
4992 m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
4993 uint64_t SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
4994 unsigned Shift = static_cast<unsigned>(ShiftAmt->Value.getZExtValue());
4995 return Shift * 2 == SrcSize;
4996 }
4997 return false;
4998}
4999
5000/// Test if the MI is shift left with half bits,
5001/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
5002static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
5003 if (MI->getOpcode() != AMDGPU::G_SHL)
5004 return false;
5005
5006 Register ShiftSrc;
5007 std::optional<ValueAndVReg> ShiftAmt;
5008 if (mi_match(MI->getOperand(0).getReg(), MRI,
5009 m_GShl(m_Reg(ShiftSrc), m_GCst(ShiftAmt)))) {
5010 uint64_t SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits();
5011 unsigned Shift = static_cast<unsigned>(ShiftAmt->Value.getZExtValue());
5012 return Shift * 2 == SrcSize;
5013 }
5014 return false;
5015}
5016
5017/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
5018static bool isUnmergeHalf(const MachineInstr *MI,
5019 const MachineRegisterInfo &MRI) {
5020 if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
5021 return false;
5022 return MI->getNumOperands() == 3 && MI->getOperand(0).isDef() &&
5023 MI->getOperand(1).isDef() && !MI->getOperand(2).isDef();
5024}
5025
5027
5029 const MachineRegisterInfo &MRI) {
5030 LLT OpTy = MRI.getType(Reg);
5031 if (OpTy.isScalar())
5032 return TypeClass::SCALAR;
5033 if (OpTy.isVector() && OpTy.getNumElements() == 2)
5036}
5037
5039 const MachineRegisterInfo &MRI) {
5041 if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
5042 return SrcStatus::INVALID;
5043
5044 switch (S) {
5045 case SrcStatus::IS_SAME:
5046 if (NegType == TypeClass::VECTOR_OF_TWO) {
5047 // Vector of 2:
5048 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5049 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5050 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5051 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5053 }
5054 if (NegType == TypeClass::SCALAR) {
5055 // Scalar:
5056 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5057 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5058 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5059 // [SrcHi, SrcLo] = [-OpHi, OpLo]
5060 return SrcStatus::IS_HI_NEG;
5061 }
5062 break;
5064 if (NegType == TypeClass::VECTOR_OF_TWO) {
5065 // Vector of 2:
5066 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5067 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5068 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5069 // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
5070 return SrcStatus::IS_LO_NEG;
5071 }
5072 if (NegType == TypeClass::SCALAR) {
5073 // Scalar:
5074 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5075 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5076 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5077 // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
5078 return SrcStatus::IS_SAME;
5079 }
5080 break;
5082 if (NegType == TypeClass::VECTOR_OF_TWO) {
5083 // Vector of 2:
5084 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5085 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5086 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5087 // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
5088 return SrcStatus::IS_HI_NEG;
5089 }
5090 if (NegType == TypeClass::SCALAR) {
5091 // Scalar:
5092 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5093 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5094 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5095 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5097 }
5098 break;
5100 if (NegType == TypeClass::VECTOR_OF_TWO) {
5101 // Vector of 2:
5102 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5103 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5104 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5105 // [SrcHi, SrcLo] = [OpHi, OpLo]
5106 return SrcStatus::IS_SAME;
5107 }
5108 if (NegType == TypeClass::SCALAR) {
5109 // Scalar:
5110 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5111 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5112 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5113 // [SrcHi, SrcLo] = [OpHi, -OpLo]
5114 return SrcStatus::IS_LO_NEG;
5115 }
5116 break;
5118 // Vector of 2:
5119 // Src = CurrUpper
5120 // Curr = [CurrUpper, CurrLower]
5121 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5122 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5123 // Src = -OpUpper
5124 //
5125 // Scalar:
5126 // Src = CurrUpper
5127 // Curr = [CurrUpper, CurrLower]
5128 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5129 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5130 // Src = -OpUpper
5133 if (NegType == TypeClass::VECTOR_OF_TWO) {
5134 // Vector of 2:
5135 // Src = CurrLower
5136 // Curr = [CurrUpper, CurrLower]
5137 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5138 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5139 // Src = -OpLower
5141 }
5142 if (NegType == TypeClass::SCALAR) {
5143 // Scalar:
5144 // Src = CurrLower
5145 // Curr = [CurrUpper, CurrLower]
5146 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5147 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5148 // Src = OpLower
5150 }
5151 break;
5153 // Vector of 2:
5154 // Src = -CurrUpper
5155 // Curr = [CurrUpper, CurrLower]
5156 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5157 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5158 // Src = -(-OpUpper) = OpUpper
5159 //
5160 // Scalar:
5161 // Src = -CurrUpper
5162 // Curr = [CurrUpper, CurrLower]
5163 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5164 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5165 // Src = -(-OpUpper) = OpUpper
5168 if (NegType == TypeClass::VECTOR_OF_TWO) {
5169 // Vector of 2:
5170 // Src = -CurrLower
5171 // Curr = [CurrUpper, CurrLower]
5172 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5173 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5174 // Src = -(-OpLower) = OpLower
5176 }
5177 if (NegType == TypeClass::SCALAR) {
5178 // Scalar:
5179 // Src = -CurrLower
5180 // Curr = [CurrUpper, CurrLower]
5181 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5182 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5183 // Src = -OpLower
5185 }
5186 break;
5187 default:
5188 break;
5189 }
5190 llvm_unreachable("unexpected SrcStatus & NegType combination");
5191}
5192
5193static std::optional<std::pair<Register, SrcStatus>>
5194calcNextStatus(std::pair<Register, SrcStatus> Curr,
5195 const MachineRegisterInfo &MRI) {
5196 const MachineInstr *MI = MRI.getVRegDef(Curr.first);
5197
5198 unsigned Opc = MI->getOpcode();
5199
5200 // Handle general Opc cases.
5201 switch (Opc) {
5202 case AMDGPU::G_BITCAST:
5203 return std::optional<std::pair<Register, SrcStatus>>(
5204 {MI->getOperand(1).getReg(), Curr.second});
5205 case AMDGPU::COPY:
5206 if (MI->getOperand(1).getReg().isPhysical())
5207 return std::nullopt;
5208 return std::optional<std::pair<Register, SrcStatus>>(
5209 {MI->getOperand(1).getReg(), Curr.second});
5210 case AMDGPU::G_FNEG: {
5211 SrcStatus Stat = getNegStatus(Curr.first, Curr.second, MRI);
5212 if (Stat == SrcStatus::INVALID)
5213 return std::nullopt;
5214 return std::optional<std::pair<Register, SrcStatus>>(
5215 {MI->getOperand(1).getReg(), Stat});
5216 }
5217 default:
5218 break;
5219 }
5220
5221 // Calc next Stat from current Stat.
5222 switch (Curr.second) {
5223 case SrcStatus::IS_SAME:
5224 if (isTruncHalf(MI, MRI))
5225 return std::optional<std::pair<Register, SrcStatus>>(
5226 {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF});
5227 else if (isUnmergeHalf(MI, MRI)) {
5228 if (Curr.first == MI->getOperand(0).getReg())
5229 return std::optional<std::pair<Register, SrcStatus>>(
5230 {MI->getOperand(2).getReg(), SrcStatus::IS_LOWER_HALF});
5231 return std::optional<std::pair<Register, SrcStatus>>(
5232 {MI->getOperand(2).getReg(), SrcStatus::IS_UPPER_HALF});
5233 }
5234 break;
5236 if (isTruncHalf(MI, MRI)) {
5237 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5238 // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
5239 // = [OpLowerHi, OpLowerLo]
5240 // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5241 // = [-OpLowerHi, OpLowerLo]
5242 // = -OpLower
5243 return std::optional<std::pair<Register, SrcStatus>>(
5244 {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5245 }
5246 if (isUnmergeHalf(MI, MRI)) {
5247 if (Curr.first == MI->getOperand(0).getReg())
5248 return std::optional<std::pair<Register, SrcStatus>>(
5249 {MI->getOperand(2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5250 return std::optional<std::pair<Register, SrcStatus>>(
5251 {MI->getOperand(2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5252 }
5253 break;
5255 if (isShlHalf(MI, MRI))
5256 return std::optional<std::pair<Register, SrcStatus>>(
5257 {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF});
5258 break;
5260 if (isLshrHalf(MI, MRI))
5261 return std::optional<std::pair<Register, SrcStatus>>(
5262 {MI->getOperand(1).getReg(), SrcStatus::IS_UPPER_HALF});
5263 break;
5265 if (isShlHalf(MI, MRI))
5266 return std::optional<std::pair<Register, SrcStatus>>(
5267 {MI->getOperand(1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5268 break;
5270 if (isLshrHalf(MI, MRI))
5271 return std::optional<std::pair<Register, SrcStatus>>(
5272 {MI->getOperand(1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5273 break;
5274 default:
5275 break;
5276 }
5277 return std::nullopt;
5278}
5279
5280/// This is used to control valid status that current MI supports. For example,
5281/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
5282/// bit on VOP3P.
5283/// The class can be further extended to recognize support on SEL, NEG, ABS bit
5284/// for different MI on different arch
5286private:
5287 bool HasNeg = false;
5288 // Assume all complex pattern of VOP3P have opsel.
5289 bool HasOpsel = true;
5290
5291public:
5293 const MachineInstr *MI = MRI.getVRegDef(Reg);
5294 unsigned Opc = MI->getOpcode();
5295
5296 if (Opc == TargetOpcode::G_INTRINSIC) {
5297 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(*MI).getIntrinsicID();
5298 // Only float point intrinsic has neg & neg_hi bits.
5299 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5300 HasNeg = true;
5302 // Keep same for generic op.
5303 HasNeg = true;
5304 }
5305 }
5306 bool checkOptions(SrcStatus Stat) const {
5307 if (!HasNeg &&
5308 (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
5309 return false;
5310 }
5311 if (!HasOpsel &&
5312 (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
5313 return false;
5314 }
5315 return true;
5316 }
5317};
5318
5321 int MaxDepth = 3) {
5322 int Depth = 0;
5323 auto Curr = calcNextStatus({Reg, SrcStatus::IS_SAME}, MRI);
5325
5326 while (Depth <= MaxDepth && Curr.has_value()) {
5327 Depth++;
5328 if (SO.checkOptions(Curr.value().second))
5329 Statlist.push_back(Curr.value());
5330 Curr = calcNextStatus(Curr.value(), MRI);
5331 }
5332
5333 return Statlist;
5334}
5335
5336static std::pair<Register, SrcStatus>
5338 int MaxDepth = 3) {
5339 int Depth = 0;
5340 std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
5341 auto Curr = calcNextStatus(LastSameOrNeg, MRI);
5342
5343 while (Depth <= MaxDepth && Curr.has_value()) {
5344 Depth++;
5345 SrcStatus Stat = Curr.value().second;
5346 if (SO.checkOptions(Stat)) {
5347 if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
5349 LastSameOrNeg = Curr.value();
5350 }
5351 Curr = calcNextStatus(Curr.value(), MRI);
5352 }
5353
5354 return LastSameOrNeg;
5355}
5356
5357static bool isSameBitWidth(Register Reg1, Register Reg2,
5358 const MachineRegisterInfo &MRI) {
5359 uint64_t Width1 = MRI.getType(Reg1).getSizeInBits();
5360 uint64_t Width2 = MRI.getType(Reg2).getSizeInBits();
5361 return Width1 == Width2;
5362}
5363
5364static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
5365 // SrcStatus::IS_LOWER_HALF remain 0.
5366 if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
5367 Mods ^= SISrcMods::NEG_HI;
5368 Mods |= SISrcMods::OP_SEL_1;
5369 } else if (HiStat == SrcStatus::IS_UPPER_HALF)
5370 Mods |= SISrcMods::OP_SEL_1;
5371 else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
5372 Mods ^= SISrcMods::NEG_HI;
5373 else if (HiStat == SrcStatus::IS_HI_NEG)
5374 Mods ^= SISrcMods::NEG_HI;
5375
5376 if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
5377 Mods ^= SISrcMods::NEG;
5378 Mods |= SISrcMods::OP_SEL_0;
5379 } else if (LoStat == SrcStatus::IS_UPPER_HALF)
5380 Mods |= SISrcMods::OP_SEL_0;
5381 else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
5382 Mods |= SISrcMods::NEG;
5383 else if (LoStat == SrcStatus::IS_HI_NEG)
5384 Mods ^= SISrcMods::NEG;
5385
5386 return Mods;
5387}
5388
5389static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
5390 Register RootReg, const SIInstrInfo &TII,
5391 const MachineRegisterInfo &MRI) {
5392 auto IsHalfState = [](SrcStatus S) {
5395 };
5396 return isSameBitWidth(NewReg, RootReg, MRI) && IsHalfState(LoStat) &&
5397 IsHalfState(HiStat);
5398}
5399
5400std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5401 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT,
5402 bool NoOpSel0) const {
5403 unsigned Mods = 0;
5404 // No modification if Root type is not form of <2 x Type>.
5405 if (isVectorOfTwoOrScalar(RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
5406 Mods |= SISrcMods::OP_SEL_1;
5407 return {RootReg, Mods};
5408 }
5409
5410 SearchOptions SO(RootReg, MRI);
5411
5412 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(RootReg, MRI, SO);
5413
5414 if (Stat.second == SrcStatus::IS_BOTH_NEG)
5416 else if (Stat.second == SrcStatus::IS_HI_NEG)
5417 Mods ^= SISrcMods::NEG_HI;
5418 else if (Stat.second == SrcStatus::IS_LO_NEG)
5419 Mods ^= SISrcMods::NEG;
5420
5421 // 64-bit VOP3P instructions do not have OPSEL or ABS. Bail on v2f64 or v2i64.
5422 // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
5423 if (MRI.getType(RootReg).getSizeInBits() == 128) {
5424 Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
5425 return {Stat.first, Mods};
5426 }
5427
5428 GBuildVector *MI;
5429 if (!mi_match(Stat.first, MRI, m_GBuildVector(MI)) ||
5430 MI->getNumOperands() != 3 || (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5431 Mods |= SISrcMods::OP_SEL_1;
5432 return {Stat.first, Mods};
5433 }
5434
5436 getSrcStats(MI->getOperand(2).getReg(), MRI, SO);
5437
5438 if (StatlistHi.empty()) {
5439 Mods |= SISrcMods::OP_SEL_1;
5440 return {Stat.first, Mods};
5441 }
5442
5444 getSrcStats(MI->getOperand(1).getReg(), MRI, SO);
5445
5446 if (StatlistLo.empty()) {
5447 Mods |= SISrcMods::OP_SEL_1;
5448 return {Stat.first, Mods};
5449 }
5450
5451 for (int I = static_cast<int>(StatlistHi.size()) - 1; I >= 0; I--) {
5452 for (int J = static_cast<int>(StatlistLo.size()) - 1; J >= 0; J--) {
5453 if (StatlistHi[I].first == StatlistLo[J].first &&
5454 isValidToPack(StatlistHi[I].second, StatlistLo[J].second,
5455 StatlistHi[I].first, RootReg, TII, MRI)) {
5456 unsigned NewMods =
5457 updateMods(StatlistHi[I].second, StatlistLo[J].second, Mods);
5458 if (NoOpSel0 && (NewMods & SISrcMods::OP_SEL_0))
5459 continue;
5460 return {StatlistHi[I].first, NewMods};
5461 }
5462 }
5463 }
5464 // Packed instructions do not have abs modifiers.
5465 Mods |= SISrcMods::OP_SEL_1;
5466
5467 return {Stat.first, Mods};
5468}
5469
5470// Removed unused function `getAllKindImm` to eliminate dead code.
5471
5472static bool checkRB(Register Reg, unsigned int RBNo,
5473 const AMDGPURegisterBankInfo &RBI,
5474 const MachineRegisterInfo &MRI,
5475 const TargetRegisterInfo &TRI) {
5476 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
5477 return RB->getID() == RBNo;
5478}
5479
5480// This function is used to get the correct register bank for returned reg.
5481// Assume:
5482// 1. VOP3P is always legal for VGPR.
5483// 2. RootOp's regbank is legal.
5484// Thus
5485// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
5486// 2. If RootOp is VGPR, then NewOp must be VGPR.
5487static Register
5490 const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
5491 // RootOp can only be VGPR or SGPR (some hand written cases such as.
5492 // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
5493 if (checkRB(RootReg, AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
5494 checkRB(NewReg, AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
5495 return NewReg;
5496
5497 if (mi_match(RootReg, MRI, m_Copy(m_SpecificReg(NewReg)))) {
5498 // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
5499 return RootReg;
5500 }
5501
5502 Register DstReg = MRI.cloneVirtualRegister(RootReg);
5503 MachineInstrBuilder MIB = BuildMI(*Use.getParent(), Use, Use.getDebugLoc(),
5504 TII.get(AMDGPU::COPY), DstReg)
5505 .addReg(NewReg);
5506
5507 // Only accept VGPR.
5508 return MIB->getOperand(0).getReg();
5509}
5510
5512AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
5513 bool IsDOT,
5514 bool NoOpSel0) const {
5515 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5516 Register Reg;
5517 unsigned Mods;
5518 std::tie(Reg, Mods) =
5519 selectVOP3PModsImpl(Root.getReg(), MRI, IsDOT, NoOpSel0);
5520
5521 return {{
5522 [=, &Root, &MRI](MachineInstrBuilder &MIB) {
5523 MIB.addReg(getLegalRegBank(Reg, Root.getReg(), *MIB.getInstr(), RBI,
5524 MRI, TRI, TII));
5525 },
5526 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
5527 }};
5528}
5529
5531AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
5532
5533 return selectVOP3PRetHelper(Root);
5534}
5535
5537AMDGPUInstructionSelector::selectVOP3PModsPkFmaOpSel0(
5538 MachineOperand &Root) const {
5539 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5540 LLT Ty = MRI.getType(Root.getReg());
5541 bool NoOpSel0 = Subtarget->hasPkF32InvalidOpSel() && Ty.isFixedVector() &&
5542 Ty.getNumElements() == 2 && Ty.getScalarSizeInBits() == 32;
5543 return selectVOP3PRetHelper(Root, /*IsDOT=*/false, NoOpSel0);
5544}
5545
5547AMDGPUInstructionSelector::selectVOP3PModsPkAddMulOpSel0(
5548 MachineOperand &Root) const {
5549 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5550 LLT Ty = MRI.getType(Root.getReg());
5551 if (Subtarget->hasPkF32InvalidOpSel() && Ty.isFixedVector() &&
5552 Ty.getNumElements() == 2 && Ty.getScalarSizeInBits() == 32 &&
5553 (selectVOP3PModsImpl(Root.getReg(), MRI).second & SISrcMods::OP_SEL_0))
5554 return std::nullopt;
5555 return selectVOP3PRetHelper(Root);
5556}
5557
5559AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
5560
5561 return selectVOP3PRetHelper(Root, true);
5562}
5563
5565AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
5566 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5567 Register Src;
5568 unsigned Mods;
5569 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true /*IsDOT*/);
5570 if (Mods != SISrcMods::OP_SEL_1)
5571 return {};
5572
5573 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); }}};
5574}
5575
5577AMDGPUInstructionSelector::selectVOP3PModsF32(MachineOperand &Root) const {
5578 Register Src;
5579 unsigned Mods;
5580 std::tie(Src, Mods) = selectVOP3PModsF32Impl(Root.getReg());
5581
5582 return {{
5583 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
5584 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
5585 }};
5586}
5587
5589AMDGPUInstructionSelector::selectVOP3PNoModsF32(MachineOperand &Root) const {
5590 Register Src;
5591 unsigned Mods;
5592 std::tie(Src, Mods) = selectVOP3PModsF32Impl(Root.getReg());
5593 if (Mods != SISrcMods::OP_SEL_1)
5594 return {};
5595
5596 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); }}};
5597}
5598
5600AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5601 MachineOperand &Root) const {
5602 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) &&
5603 "expected i1 value");
5604 unsigned Mods = SISrcMods::OP_SEL_1;
5605 if (Root.getImm() != 0)
5606 Mods |= SISrcMods::OP_SEL_0;
5607
5608 return {{
5609 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
5610 }};
5611}
5612
5614 MachineInstr *InsertPt,
5616 const TargetRegisterClass *DstRegClass;
5617 switch (Elts.size()) {
5618 case 8:
5619 DstRegClass = &AMDGPU::VReg_256RegClass;
5620 break;
5621 case 4:
5622 DstRegClass = &AMDGPU::VReg_128RegClass;
5623 break;
5624 case 2:
5625 DstRegClass = &AMDGPU::VReg_64RegClass;
5626 break;
5627 default:
5628 llvm_unreachable("unhandled Reg sequence size");
5629 }
5630
5631 MachineIRBuilder B(*InsertPt);
5632 auto MIB = B.buildInstr(AMDGPU::REG_SEQUENCE)
5633 .addDef(MRI.createVirtualRegister(DstRegClass));
5634 for (unsigned i = 0; i < Elts.size(); ++i) {
5635 MIB.addReg(Elts[i]);
5637 }
5638 return MIB->getOperand(0).getReg();
5639}
5640
5641static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
5643 MachineInstr *InsertPt,
5645 if (ModOpcode == TargetOpcode::G_FNEG) {
5646 Mods |= SISrcMods::NEG;
5647 // Check if all elements also have abs modifier
5648 SmallVector<Register, 8> NegAbsElts;
5649 for (auto El : Elts) {
5650 Register FabsSrc;
5651 if (!mi_match(El, MRI, m_GFabs(m_Reg(FabsSrc))))
5652 break;
5653 NegAbsElts.push_back(FabsSrc);
5654 }
5655 if (Elts.size() != NegAbsElts.size()) {
5656 // Neg
5657 Src = buildRegSequence(Elts, InsertPt, MRI);
5658 } else {
5659 // Neg and Abs
5660 Mods |= SISrcMods::NEG_HI;
5661 Src = buildRegSequence(NegAbsElts, InsertPt, MRI);
5662 }
5663 } else {
5664 assert(ModOpcode == TargetOpcode::G_FABS);
5665 // Abs
5666 Mods |= SISrcMods::NEG_HI;
5667 Src = buildRegSequence(Elts, InsertPt, MRI);
5668 }
5669}
5670
5672AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const {
5673 Register Src = Root.getReg();
5674 unsigned Mods = SISrcMods::OP_SEL_1;
5676
5677 GBuildVector *BV;
5678 if (mi_match(Src, *MRI, m_GBuildVector(BV))) {
5679 assert(BV->getNumSources() > 0);
5680 // Based on first element decide which mod we match, neg or abs
5681 MachineInstr *ElF32 = MRI->getVRegDef(BV->getSourceReg(0));
5682 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG)
5683 ? AMDGPU::G_FNEG
5684 : AMDGPU::G_FABS;
5685 for (unsigned i = 0; i < BV->getNumSources(); ++i) {
5686 ElF32 = MRI->getVRegDef(BV->getSourceReg(i));
5687 if (ElF32->getOpcode() != ModOpcode)
5688 break;
5689 EltsF32.push_back(ElF32->getOperand(1).getReg());
5690 }
5691
5692 // All elements had ModOpcode modifier
5693 if (BV->getNumSources() == EltsF32.size()) {
5694 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF32, Src, Root.getParent(),
5695 *MRI);
5696 }
5697 }
5698
5699 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
5700 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }}};
5701}
5702
5704AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const {
5705 Register Src = Root.getReg();
5706 unsigned Mods = SISrcMods::OP_SEL_1;
5707 SmallVector<Register, 8> EltsV2F16;
5708
5709 GConcatVectors *CV;
5710 if (mi_match(Src, *MRI, m_GConcatVectors(CV))) {
5711 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5712 Register FNegSrc;
5713 if (!mi_match(CV->getSourceReg(i), *MRI, m_GFNeg(m_Reg(FNegSrc))))
5714 break;
5715 EltsV2F16.push_back(FNegSrc);
5716 }
5717
5718 // All elements had ModOpcode modifier
5719 if (CV->getNumSources() == EltsV2F16.size()) {
5720 Mods |= SISrcMods::NEG;
5721 Mods |= SISrcMods::NEG_HI;
5722 Src = buildRegSequence(EltsV2F16, Root.getParent(), *MRI);
5723 }
5724 }
5725
5726 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
5727 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }}};
5728}
5729
5731AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const {
5732 Register Src = Root.getReg();
5733 unsigned Mods = SISrcMods::OP_SEL_1;
5734 SmallVector<Register, 8> EltsV2F16;
5735
5736 GConcatVectors *CV;
5737 if (mi_match(Src, *MRI, m_GConcatVectors(CV))) {
5738 assert(CV->getNumSources() > 0);
5739 MachineInstr *ElV2F16 = MRI->getVRegDef(CV->getSourceReg(0));
5740 // Based on first element decide which mod we match, neg or abs
5741 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG)
5742 ? AMDGPU::G_FNEG
5743 : AMDGPU::G_FABS;
5744
5745 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5746 ElV2F16 = MRI->getVRegDef(CV->getSourceReg(i));
5747 if (ElV2F16->getOpcode() != ModOpcode)
5748 break;
5749 EltsV2F16.push_back(ElV2F16->getOperand(1).getReg());
5750 }
5751
5752 // All elements had ModOpcode modifier
5753 if (CV->getNumSources() == EltsV2F16.size()) {
5754 selectWMMAModsNegAbs(ModOpcode, Mods, EltsV2F16, Src, Root.getParent(),
5755 *MRI);
5756 }
5757 }
5758
5759 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
5760 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }}};
5761}
5762
5764AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const {
5765 std::optional<FPValueAndVReg> FPValReg;
5766 if (mi_match(Root.getReg(), *MRI, m_GFCstOrSplat(FPValReg))) {
5767 if (TII.isInlineConstant(FPValReg->Value)) {
5768 return {{[=](MachineInstrBuilder &MIB) {
5769 MIB.addImm(FPValReg->Value.bitcastToAPInt().getSExtValue());
5770 }}};
5771 }
5772 // Non-inlineable splat floats should not fall-through for integer immediate
5773 // checks.
5774 return {};
5775 }
5776
5777 APInt ICst;
5778 if (mi_match(Root.getReg(), *MRI, m_ICstOrSplat(ICst))) {
5779 if (TII.isInlineConstant(ICst)) {
5780 return {
5781 {[=](MachineInstrBuilder &MIB) { MIB.addImm(ICst.getSExtValue()); }}};
5782 }
5783 }
5784
5785 return {};
5786}
5787
5789AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const {
5790 Register Src =
5791 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg();
5792 unsigned Key = 0;
5793
5794 Register ShiftSrc;
5795 std::optional<ValueAndVReg> ShiftAmt;
5796 if (mi_match(Src, *MRI, m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt))) &&
5797 MRI->getType(ShiftSrc).getSizeInBits() == 32 &&
5798 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5799 Key = static_cast<unsigned>(ShiftAmt->Value.getZExtValue() / 8);
5800 Src = ShiftSrc;
5801 }
5802
5803 return {{
5804 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
5805 [=](MachineInstrBuilder &MIB) { MIB.addImm(Key); } // index_key
5806 }};
5807}
5808
5810AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const {
5811
5812 Register Src =
5813 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg();
5814 unsigned Key = 0;
5815
5816 Register ShiftSrc;
5817 std::optional<ValueAndVReg> ShiftAmt;
5818 if (mi_match(Src, *MRI, m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt))) &&
5819 MRI->getType(ShiftSrc).getSizeInBits() == 32 &&
5820 ShiftAmt->Value.getZExtValue() == 16) {
5821 Src = ShiftSrc;
5822 Key = 1;
5823 }
5824
5825 return {{
5826 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
5827 [=](MachineInstrBuilder &MIB) { MIB.addImm(Key); } // index_key
5828 }};
5829}
5830
5832AMDGPUInstructionSelector::selectSWMMACIndex32(MachineOperand &Root) const {
5833 Register Src =
5834 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg();
5835 unsigned Key = 0;
5836
5837 Register S32 = matchZeroExtendFromS32(Src);
5838 if (!S32)
5839 S32 = matchAnyExtendFromS32(Src);
5840
5841 if (S32) {
5842 const MachineInstr *Def = getDefIgnoringCopies(S32, *MRI);
5843 if (Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5844 assert(Def->getNumOperands() == 3);
5845 Register DstReg1 = Def->getOperand(1).getReg();
5846 if (mi_match(S32, *MRI,
5847 m_any_of(m_SpecificReg(DstReg1), m_Copy(m_Reg(DstReg1))))) {
5848 Src = Def->getOperand(2).getReg();
5849 Key = 1;
5850 }
5851 }
5852 }
5853
5854 return {{
5855 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
5856 [=](MachineInstrBuilder &MIB) { MIB.addImm(Key); } // index_key
5857 }};
5858}
5859
5861AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
5862 Register Src;
5863 unsigned Mods;
5864 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg());
5865
5866 Register ExtractSrc;
5867 if (!Subtarget->useRealTrue16Insts() &&
5868 MRI->getType(Root.getReg()).getSizeInBits() == 16 &&
5869 isExtractHiElt(*MRI, Src, ExtractSrc)) {
5870 Src = ExtractSrc;
5871 Mods |= SISrcMods::OP_SEL_0;
5872 }
5873
5874 return {{
5875 [=](MachineInstrBuilder &MIB) {
5876 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
5877 },
5878 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
5879 }};
5880}
5881
5882// FIXME-TRUE16 remove when fake16 is removed
5884AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
5885 Register Src;
5886 unsigned Mods;
5887 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(),
5888 /*IsCanonicalizing=*/true,
5889 /*AllowAbs=*/false,
5890 /*OpSel=*/false);
5891
5892 return {{
5893 [=](MachineInstrBuilder &MIB) {
5894 MIB.addReg(
5895 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true));
5896 },
5897 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
5898 }};
5899}
5900
5902AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
5903 Register Src;
5904 unsigned Mods;
5905 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(),
5906 /*IsCanonicalizing=*/true,
5907 /*AllowAbs=*/false,
5908 /*OpSel=*/true);
5909
5910 return {{
5911 [=](MachineInstrBuilder &MIB) {
5912 MIB.addReg(
5913 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true));
5914 },
5915 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods
5916 }};
5917}
5918
5919// Given \p Offset and load specified by the \p Root operand check if \p Offset
5920// is a multiple of the load byte size. If it is update \p Offset to a
5921// pre-scaled value and return true.
5922bool AMDGPUInstructionSelector::selectScaleOffset(MachineOperand &Root,
5924 bool IsSigned,
5925 bool IsSMEM) const {
5926 if (!Subtarget->hasScaleOffset())
5927 return false;
5928
5929 const MachineInstr &MI = *Root.getParent();
5930 MachineMemOperand *MMO = *MI.memoperands_begin();
5931
5932 if (!MMO->getSize().hasValue())
5933 return false;
5934
5935 uint64_t Size = MMO->getSize().getValue();
5936
5937 Register OffsetReg = matchExtendFromS32OrS32(Offset, IsSigned);
5938 bool IsExtended = OffsetReg && OffsetReg != Offset;
5939 if (!OffsetReg)
5940 OffsetReg = Offset;
5941
5942 if (auto Def = getDefSrcRegIgnoringCopies(OffsetReg, *MRI))
5943 OffsetReg = Def->Reg;
5944
5945 Register Op0;
5946 MachineInstr *Mul;
5947 bool ScaleOffset =
5948 (isPowerOf2_64(Size) &&
5949 mi_match(OffsetReg, *MRI,
5950 m_GShl(m_Reg(Op0),
5953 mi_match(OffsetReg, *MRI,
5955 m_Copy(m_SpecificICst(Size))))) ||
5956 mi_match(
5957 OffsetReg, *MRI,
5958 m_BinOp(IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5959 m_Reg(Op0), m_SpecificICst(Size))) ||
5960 // Match G_AMDGPU_MAD_U64_U32 offset, c, 0
5961 (mi_match(OffsetReg, *MRI, m_MInstr(Mul)) &&
5962 (Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5963 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5964 (IsSigned && Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5965 VT->signBitIsZero(Mul->getOperand(2).getReg()))) &&
5966 mi_match(Mul->getOperand(4).getReg(), *MRI, m_ZeroInt()) &&
5967 mi_match(Mul->getOperand(3).getReg(), *MRI,
5969 m_Copy(m_SpecificICst(Size))))) &&
5970 mi_match(Mul->getOperand(2).getReg(), *MRI, m_Reg(Op0)));
5971
5972 if (!ScaleOffset)
5973 return false;
5974
5975 // SMEM scales the offset in 32 bits, other instructions in 64 bits.
5976 unsigned Bits = Log2_64_Ceil(Size);
5977 if (IsSMEM) {
5978 if (!IsExtended && MRI->getType(Offset).getSizeInBits() == 64) {
5979 KnownBits Known = VT->getKnownBits(Op0);
5980 if (Known.countMaxActiveBits() + Bits > 32)
5981 return false;
5982 }
5983 } else if (IsExtended) {
5984 const MachineInstr *Scale = MRI->getVRegDef(OffsetReg);
5985 if (IsSigned && !Scale->getFlag(MachineInstr::NoSWrap) &&
5986 VT->computeNumSignBits(Op0) <= Bits)
5987 return false;
5988 if (!IsSigned && !Scale->getFlag(MachineInstr::NoUWrap) &&
5989 VT->getKnownBits(Op0).countMinLeadingZeros() < Bits)
5990 return false;
5991 }
5992
5993 Offset = Op0;
5994 return true;
5995}
5996
5997bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
5998 Register &Base,
5999 Register *SOffset,
6000 int64_t *Offset,
6001 bool *ScaleOffset) const {
6002 MachineInstr *MI = Root.getParent();
6003 MachineBasicBlock *MBB = MI->getParent();
6004
6005 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
6006 // then we can select all ptr + 32-bit offsets.
6007 SmallVector<GEPInfo, 4> AddrInfo;
6008 getAddrModeInfo(*MI, *MRI, AddrInfo);
6009
6010 if (AddrInfo.empty())
6011 return false;
6012
6013 const GEPInfo &GEPI = AddrInfo[0];
6014 std::optional<int64_t> EncodedImm;
6015
6016 if (ScaleOffset)
6017 *ScaleOffset = false;
6018
6019 if (SOffset && Offset) {
6020 EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPI.Imm, /*IsBuffer=*/false,
6021 /*HasSOffset=*/true);
6022 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
6023 AddrInfo.size() > 1) {
6024 const GEPInfo &GEPI2 = AddrInfo[1];
6025 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
6026 Register OffsetReg = GEPI2.SgprParts[1];
6027 if (ScaleOffset)
6028 *ScaleOffset = selectScaleOffset(
6029 Root, OffsetReg, false /* IsSigned */, true /* IsSMEM */);
6030 OffsetReg = matchZeroExtendFromS32OrS32(OffsetReg);
6031 if (OffsetReg) {
6032 Base = GEPI2.SgprParts[0];
6033 *SOffset = OffsetReg;
6034 *Offset = *EncodedImm;
6035 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(STI))
6036 return true;
6037
6038 // For unbuffered smem loads, it is illegal for the Immediate Offset
6039 // to be negative if the resulting (Offset + (M0 or SOffset or zero)
6040 // is negative. Handle the case where the Immediate Offset + SOffset
6041 // is negative.
6042 auto SKnown = VT->getKnownBits(*SOffset);
6043 if (*Offset + SKnown.getMinValue().getSExtValue() < 0)
6044 return false;
6045
6046 return true;
6047 }
6048 }
6049 }
6050 return false;
6051 }
6052
6053 EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPI.Imm, /*IsBuffer=*/false,
6054 /*HasSOffset=*/false);
6055 if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
6056 Base = GEPI.SgprParts[0];
6057 *Offset = *EncodedImm;
6058 return true;
6059 }
6060
6061 // SGPR offset is unsigned.
6062 if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(GEPI.Imm) &&
6063 GEPI.Imm != 0) {
6064 // If we make it this far we have a load with an 32-bit immediate offset.
6065 // It is OK to select this using a sgpr offset, because we have already
6066 // failed trying to select this load into one of the _IMM variants since
6067 // the _IMM Patterns are considered before the _SGPR patterns.
6068 Base = GEPI.SgprParts[0];
6069 *SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
6070 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), *SOffset)
6071 .addImm(GEPI.Imm);
6072 return true;
6073 }
6074
6075 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
6076 Register OffsetReg = GEPI.SgprParts[1];
6077 if (ScaleOffset)
6078 *ScaleOffset = selectScaleOffset(Root, OffsetReg, false /* IsSigned */,
6079 true /* IsSMEM */);
6080 OffsetReg = matchZeroExtendFromS32OrS32(OffsetReg);
6081 if (OffsetReg) {
6082 Base = GEPI.SgprParts[0];
6083 *SOffset = OffsetReg;
6084 return true;
6085 }
6086 }
6087
6088 return false;
6089}
6090
6092AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
6093 Register Base;
6094 int64_t Offset;
6095 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, &Offset,
6096 /* ScaleOffset */ nullptr))
6097 return std::nullopt;
6098
6099 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Base); },
6100 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }}};
6101}
6102
6104AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
6105 SmallVector<GEPInfo, 4> AddrInfo;
6106 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo);
6107
6108 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
6109 return std::nullopt;
6110
6111 const GEPInfo &GEPInfo = AddrInfo[0];
6112 Register PtrReg = GEPInfo.SgprParts[0];
6113 std::optional<int64_t> EncodedImm =
6114 AMDGPU::getSMRDEncodedLiteralOffset32(STI, GEPInfo.Imm);
6115 if (!EncodedImm)
6116 return std::nullopt;
6117
6118 return {{
6119 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); },
6120 [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); }
6121 }};
6122}
6123
6125AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
6126 Register Base, SOffset;
6127 bool ScaleOffset;
6128 if (!selectSmrdOffset(Root, Base, &SOffset, /* Offset= */ nullptr,
6129 &ScaleOffset))
6130 return std::nullopt;
6131
6132 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6133 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Base); },
6134 [=](MachineInstrBuilder &MIB) { MIB.addReg(SOffset); },
6135 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPol); }}};
6136}
6137
6139AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
6140 Register Base, SOffset;
6141 int64_t Offset;
6142 bool ScaleOffset;
6143 if (!selectSmrdOffset(Root, Base, &SOffset, &Offset, &ScaleOffset))
6144 return std::nullopt;
6145
6146 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6147 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Base); },
6148 [=](MachineInstrBuilder &MIB) { MIB.addReg(SOffset); },
6149 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); },
6150 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPol); }}};
6151}
6152
6153std::pair<Register, int64_t> AMDGPUInstructionSelector::selectFlatOffsetImpl(
6154 MachineOperand &Root, AMDGPU::FlatAddrSpace FlatVariant) const {
6155 MachineInstr *MI = Root.getParent();
6156
6157 auto Default = std::pair(Root.getReg(), 0);
6158
6159 if (!STI.hasFlatInstOffsets())
6160 return Default;
6161
6162 Register PtrBase;
6163 int64_t ConstOffset;
6164 bool IsInBounds;
6165 std::tie(PtrBase, ConstOffset, IsInBounds) =
6166 getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
6167
6168 // Adding the offset to the base address with an immediate in a FLAT
6169 // instruction must not change the memory aperture in which the address falls.
6170 // Therefore we can only fold offsets from inbounds GEPs into FLAT
6171 // instructions.
6172 if (ConstOffset == 0 ||
6173 (FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch &&
6174 !isFlatScratchBaseLegal(Root.getReg())) ||
6175 (FlatVariant == AMDGPU::FlatAddrSpace::FLAT && !IsInBounds))
6176 return Default;
6177
6178 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
6179 if (!TII.isLegalFLATOffset(ConstOffset, AddrSpace, FlatVariant))
6180 return Default;
6181
6182 return std::pair(PtrBase, ConstOffset);
6183}
6184
6186AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
6187 auto PtrWithOffset = selectFlatOffsetImpl(Root, AMDGPU::FlatAddrSpace::FLAT);
6188
6189 return {{
6190 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrWithOffset.first); },
6191 [=](MachineInstrBuilder &MIB) { MIB.addImm(PtrWithOffset.second); },
6192 }};
6193}
6194
6196AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
6197 auto PtrWithOffset =
6198 selectFlatOffsetImpl(Root, AMDGPU::FlatAddrSpace::FlatGlobal);
6199
6200 return {{
6201 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrWithOffset.first); },
6202 [=](MachineInstrBuilder &MIB) { MIB.addImm(PtrWithOffset.second); },
6203 }};
6204}
6205
6207AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
6208 auto PtrWithOffset =
6209 selectFlatOffsetImpl(Root, AMDGPU::FlatAddrSpace::FlatScratch);
6210
6211 return {{
6212 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrWithOffset.first); },
6213 [=](MachineInstrBuilder &MIB) { MIB.addImm(PtrWithOffset.second); },
6214 }};
6215}
6216
6217// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
6219AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
6220 unsigned CPolBits,
6221 bool NeedIOffset) const {
6222 Register Addr = Root.getReg();
6223 Register PtrBase;
6224 int64_t ConstOffset;
6225 int64_t ImmOffset = 0;
6226
6227 // Match the immediate offset first, which canonically is moved as low as
6228 // possible.
6229 std::tie(PtrBase, ConstOffset, std::ignore) =
6230 getPtrBaseWithConstantOffset(Addr, *MRI);
6231
6232 if (ConstOffset != 0) {
6233 if (NeedIOffset &&
6234 TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::GLOBAL_ADDRESS,
6236 Addr = PtrBase;
6237 ImmOffset = ConstOffset;
6238 } else {
6239 auto PtrBaseDef = getDefSrcRegIgnoringCopies(PtrBase, *MRI);
6240 if (isSGPR(PtrBaseDef->Reg)) {
6241 // Offset is too large.
6242 //
6243 // saddr + large_offset -> saddr +
6244 // (voffset = large_offset & ~MaxOffset) +
6245 // (large_offset & MaxOffset);
6246 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6247 if (NeedIOffset) {
6248 std::tie(SplitImmOffset, RemainderOffset) =
6249 TII.splitFlatOffset(ConstOffset, AMDGPUAS::GLOBAL_ADDRESS,
6251 }
6252
6253 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(RemainderOffset)
6254 : isUInt<32>(RemainderOffset)) {
6255 MachineInstr *MI = Root.getParent();
6256 MachineBasicBlock *MBB = MI->getParent();
6257 Register HighBits =
6258 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6259
6260 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
6261 HighBits)
6262 .addImm(RemainderOffset);
6263
6264 if (NeedIOffset)
6265 return {{
6266 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrBase); }, // saddr
6267 [=](MachineInstrBuilder &MIB) {
6268 MIB.addReg(HighBits);
6269 }, // voffset
6270 [=](MachineInstrBuilder &MIB) { MIB.addImm(SplitImmOffset); },
6271 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPolBits); },
6272 }};
6273 return {{
6274 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrBase); }, // saddr
6275 [=](MachineInstrBuilder &MIB) {
6276 MIB.addReg(HighBits);
6277 }, // voffset
6278 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPolBits); },
6279 }};
6280 }
6281
6282 // We are adding a 64 bit SGPR and a constant. If constant bus limit
6283 // is 1 we would need to perform 1 or 2 extra moves for each half of
6284 // the constant and it is better to do a scalar add and then issue a
6285 // single VALU instruction to materialize zero. Otherwise it is less
6286 // instructions to perform VALU adds with immediates or inline literals.
6287 unsigned NumLiterals =
6288 !TII.isInlineConstant(APInt(32, Lo_32(ConstOffset))) +
6289 !TII.isInlineConstant(APInt(32, Hi_32(ConstOffset)));
6290 if (STI.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
6291 return std::nullopt;
6292 }
6293 }
6294 }
6295
6296 // Match the variable offset.
6297 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
6298 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6299 // Look through the SGPR->VGPR copy.
6300 Register SAddr =
6301 getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI);
6302
6303 if (isSGPR(SAddr)) {
6304 Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
6305
6306 // It's possible voffset is an SGPR here, but the copy to VGPR will be
6307 // inserted later.
6308 bool ScaleOffset = selectScaleOffset(Root, PtrBaseOffset,
6309 Subtarget->hasSignedGVSOffset());
6310 if (Register VOffset = matchExtendFromS32OrS32(
6311 PtrBaseOffset, Subtarget->hasSignedGVSOffset())) {
6312 if (NeedIOffset)
6313 return {{[=](MachineInstrBuilder &MIB) { // saddr
6314 MIB.addReg(SAddr);
6315 },
6316 [=](MachineInstrBuilder &MIB) { // voffset
6317 MIB.addReg(VOffset);
6318 },
6319 [=](MachineInstrBuilder &MIB) { // offset
6320 MIB.addImm(ImmOffset);
6321 },
6322 [=](MachineInstrBuilder &MIB) { // cpol
6323 MIB.addImm(CPolBits |
6324 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6325 }}};
6326 return {{[=](MachineInstrBuilder &MIB) { // saddr
6327 MIB.addReg(SAddr);
6328 },
6329 [=](MachineInstrBuilder &MIB) { // voffset
6330 MIB.addReg(VOffset);
6331 },
6332 [=](MachineInstrBuilder &MIB) { // cpol
6333 MIB.addImm(CPolBits |
6334 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6335 }}};
6336 }
6337 }
6338 }
6339
6340 // FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
6341 // drop this.
6342 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6343 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(AddrDef->Reg))
6344 return std::nullopt;
6345
6346 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
6347 // moves required to copy a 64-bit SGPR to VGPR.
6348 MachineInstr *MI = Root.getParent();
6349 MachineBasicBlock *MBB = MI->getParent();
6350 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6351
6352 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), VOffset)
6353 .addImm(0);
6354
6355 if (NeedIOffset)
6356 return {{
6357 [=](MachineInstrBuilder &MIB) { MIB.addReg(AddrDef->Reg); }, // saddr
6358 [=](MachineInstrBuilder &MIB) { MIB.addReg(VOffset); }, // voffset
6359 [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); }, // offset
6360 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPolBits); } // cpol
6361 }};
6362 return {{
6363 [=](MachineInstrBuilder &MIB) { MIB.addReg(AddrDef->Reg); }, // saddr
6364 [=](MachineInstrBuilder &MIB) { MIB.addReg(VOffset); }, // voffset
6365 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPolBits); } // cpol
6366 }};
6367}
6368
6370AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
6371 return selectGlobalSAddr(Root, 0);
6372}
6373
6375AMDGPUInstructionSelector::selectGlobalSAddrCPol(MachineOperand &Root) const {
6376 const MachineInstr &I = *Root.getParent();
6377
6378 // We are assuming CPol is always the last operand of the intrinsic.
6379 auto PassedCPol =
6380 I.getOperand(I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6381 return selectGlobalSAddr(Root, static_cast<unsigned>(PassedCPol));
6382}
6383
6385AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(MachineOperand &Root) const {
6386 const MachineInstr &I = *Root.getParent();
6387
6388 // We are assuming CPol is second from last operand of the intrinsic.
6389 auto PassedCPol =
6390 I.getOperand(I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6391 return selectGlobalSAddr(Root, static_cast<unsigned>(PassedCPol));
6392}
6393
6395AMDGPUInstructionSelector::selectGlobalSAddrGLC(MachineOperand &Root) const {
6396 return selectGlobalSAddr(Root, AMDGPU::CPol::GLC);
6397}
6398
6400AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6401 MachineOperand &Root) const {
6402 const MachineInstr &I = *Root.getParent();
6403
6404 // We are assuming CPol is always the last operand of the intrinsic.
6405 auto PassedCPol =
6406 I.getOperand(I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6407 return selectGlobalSAddr(Root, static_cast<unsigned>(PassedCPol), false);
6408}
6409
6411AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6412 MachineOperand &Root) const {
6413 const MachineInstr &I = *Root.getParent();
6414
6415 // We are assuming CPol is second from last operand of the intrinsic.
6416 auto PassedCPol =
6417 I.getOperand(I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6418 return selectGlobalSAddr(Root, static_cast<unsigned>(PassedCPol), false);
6419}
6420
6422AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
6423 Register Addr = Root.getReg();
6424 Register PtrBase;
6425 int64_t ConstOffset;
6426 int64_t ImmOffset = 0;
6427
6428 // Match the immediate offset first, which canonically is moved as low as
6429 // possible.
6430 std::tie(PtrBase, ConstOffset, std::ignore) =
6431 getPtrBaseWithConstantOffset(Addr, *MRI);
6432
6433 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6434 TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS,
6436 Addr = PtrBase;
6437 ImmOffset = ConstOffset;
6438 }
6439
6440 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
6441 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6442 int FI = AddrDef->MI->getOperand(1).getIndex();
6443 return {{
6444 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); }, // saddr
6445 [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
6446 }};
6447 }
6448
6449 Register SAddr = AddrDef->Reg;
6450
6451 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6452 Register LHS = AddrDef->MI->getOperand(1).getReg();
6453 Register RHS = AddrDef->MI->getOperand(2).getReg();
6454 auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI);
6455 auto RHSDef = getDefSrcRegIgnoringCopies(RHS, *MRI);
6456
6457 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6458 isSGPR(RHSDef->Reg)) {
6459 int FI = LHSDef->MI->getOperand(1).getIndex();
6460 MachineInstr &I = *Root.getParent();
6461 MachineBasicBlock *BB = I.getParent();
6462 const DebugLoc &DL = I.getDebugLoc();
6463 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
6464
6465 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_I32), SAddr)
6466 .addFrameIndex(FI)
6467 .addReg(RHSDef->Reg)
6468 .setOperandDead(3); // Dead scc
6469 }
6470 }
6471
6472 if (!isSGPR(SAddr))
6473 return std::nullopt;
6474
6475 return {{
6476 [=](MachineInstrBuilder &MIB) { MIB.addReg(SAddr); }, // saddr
6477 [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); } // offset
6478 }};
6479}
6480
6481// Check whether the flat scratch SVS swizzle bug affects this access.
6482bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6483 Register VAddr, Register SAddr, uint64_t ImmOffset) const {
6484 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6485 return false;
6486
6487 // The bug affects the swizzling of SVS accesses if there is any carry out
6488 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
6489 // voffset to (soffset + inst_offset).
6490 auto VKnown = VT->getKnownBits(VAddr);
6491 auto SKnown = KnownBits::add(VT->getKnownBits(SAddr),
6492 KnownBits::makeConstant(APInt(32, ImmOffset)));
6493 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6494 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
6495 return (VMax & 3) + (SMax & 3) >= 4;
6496}
6497
6499AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
6500 Register Addr = Root.getReg();
6501 Register PtrBase;
6502 int64_t ConstOffset;
6503 int64_t ImmOffset = 0;
6504
6505 // Match the immediate offset first, which canonically is moved as low as
6506 // possible.
6507 std::tie(PtrBase, ConstOffset, std::ignore) =
6508 getPtrBaseWithConstantOffset(Addr, *MRI);
6509
6510 Register OrigAddr = Addr;
6511 if (ConstOffset != 0 &&
6512 TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS,
6514 Addr = PtrBase;
6515 ImmOffset = ConstOffset;
6516 }
6517
6518 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI);
6519 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6520 return std::nullopt;
6521
6522 Register RHS = AddrDef->MI->getOperand(2).getReg();
6523 if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6524 return std::nullopt;
6525
6526 Register LHS = AddrDef->MI->getOperand(1).getReg();
6527 auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI);
6528
6529 if (OrigAddr != Addr) {
6530 if (!isFlatScratchBaseLegalSVImm(OrigAddr))
6531 return std::nullopt;
6532 } else {
6533 if (!isFlatScratchBaseLegalSV(OrigAddr))
6534 return std::nullopt;
6535 }
6536
6537 if (checkFlatScratchSVSSwizzleBug(RHS, LHS, ImmOffset))
6538 return std::nullopt;
6539
6540 unsigned CPol = selectScaleOffset(Root, RHS, true /* IsSigned */)
6542 : 0;
6543
6544 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6545 int FI = LHSDef->MI->getOperand(1).getIndex();
6546 return {{
6547 [=](MachineInstrBuilder &MIB) { MIB.addReg(RHS); }, // vaddr
6548 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); }, // saddr
6549 [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); }, // offset
6550 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPol); } // cpol
6551 }};
6552 }
6553
6554 if (!isSGPR(LHS))
6555 if (auto Def = getDefSrcRegIgnoringCopies(LHS, *MRI))
6556 LHS = Def->Reg;
6557
6558 if (!isSGPR(LHS))
6559 return std::nullopt;
6560
6561 return {{
6562 [=](MachineInstrBuilder &MIB) { MIB.addReg(RHS); }, // vaddr
6563 [=](MachineInstrBuilder &MIB) { MIB.addReg(LHS); }, // saddr
6564 [=](MachineInstrBuilder &MIB) { MIB.addImm(ImmOffset); }, // offset
6565 [=](MachineInstrBuilder &MIB) { MIB.addImm(CPol); } // cpol
6566 }};
6567}
6568
6570AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
6571 MachineInstr *MI = Root.getParent();
6572 MachineBasicBlock *MBB = MI->getParent();
6574 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6575
6576 int64_t Offset = 0;
6577 if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) &&
6579 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6580
6581 // TODO: Should this be inside the render function? The iterator seems to
6582 // move.
6583 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(*Subtarget);
6584 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
6585 HighBits)
6586 .addImm(Offset & ~MaxOffset);
6587
6588 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6589 MIB.addReg(Info->getScratchRSrcReg());
6590 },
6591 [=](MachineInstrBuilder &MIB) { // vaddr
6592 MIB.addReg(HighBits);
6593 },
6594 [=](MachineInstrBuilder &MIB) { // soffset
6595 // Use constant zero for soffset and rely on eliminateFrameIndex
6596 // to choose the appropriate frame register if need be.
6597 MIB.addImm(0);
6598 },
6599 [=](MachineInstrBuilder &MIB) { // offset
6600 MIB.addImm(Offset & MaxOffset);
6601 }}};
6602 }
6603
6604 assert(Offset == 0 || Offset == -1);
6605
6606 // Try to fold a frame index directly into the MUBUF vaddr field, and any
6607 // offsets.
6608 std::optional<int> FI;
6609 Register VAddr = Root.getReg();
6610
6611 Register PtrBase;
6612 int64_t ConstOffset;
6613 std::tie(PtrBase, ConstOffset, std::ignore) =
6614 getPtrBaseWithConstantOffset(VAddr, *MRI);
6615 int MatchedFI;
6616 if (ConstOffset != 0) {
6617 if (TII.isLegalMUBUFImmOffset(ConstOffset) &&
6618 (!STI.privateMemoryResourceIsRangeChecked() ||
6619 VT->signBitIsZero(PtrBase))) {
6620 if (mi_match(PtrBase, *MRI, m_GFrameIndex(MatchedFI)))
6621 FI = MatchedFI;
6622 else
6623 VAddr = PtrBase;
6624 Offset = ConstOffset;
6625 }
6626 } else if (mi_match(Root.getReg(), *MRI, m_GFrameIndex(MatchedFI))) {
6627 FI = MatchedFI;
6628 }
6629
6630 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6631 MIB.addReg(Info->getScratchRSrcReg());
6632 },
6633 [=](MachineInstrBuilder &MIB) { // vaddr
6634 if (FI)
6635 MIB.addFrameIndex(*FI);
6636 else
6637 MIB.addReg(VAddr);
6638 },
6639 [=](MachineInstrBuilder &MIB) { // soffset
6640 // Use constant zero for soffset and rely on eliminateFrameIndex
6641 // to choose the appropriate frame register if need be.
6642 MIB.addImm(0);
6643 },
6644 [=](MachineInstrBuilder &MIB) { // offset
6645 MIB.addImm(Offset);
6646 }}};
6647}
6648
6649bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
6650 int64_t Offset) const {
6651 if (!isUInt<16>(Offset))
6652 return false;
6653
6654 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6655 return true;
6656
6657 // On Southern Islands instruction with a negative base value and an offset
6658 // don't seem to work.
6659 return VT->signBitIsZero(Base);
6660}
6661
6662bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
6663 int64_t Offset1,
6664 unsigned Size) const {
6665 if (Offset0 % Size != 0 || Offset1 % Size != 0)
6666 return false;
6667 if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size))
6668 return false;
6669
6670 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6671 return true;
6672
6673 // On Southern Islands instruction with a negative base value and an offset
6674 // don't seem to work.
6675 return VT->signBitIsZero(Base);
6676}
6677
6678// Return whether the operation has NoUnsignedWrap property.
6679static bool isNoUnsignedWrap(MachineInstr *Addr) {
6680 return Addr->getOpcode() == TargetOpcode::G_OR ||
6681 (Addr->getOpcode() == TargetOpcode::G_PTR_ADD &&
6683}
6684
6685// Check that the base address of flat scratch load/store in the form of `base +
6686// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
6687// requirement). We always treat the first operand as the base address here.
6688bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {
6689 MachineInstr *AddrMI = getDefIgnoringCopies(Addr, *MRI);
6690
6691 if (isNoUnsignedWrap(AddrMI))
6692 return true;
6693
6694 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6695 // values.
6696 if (STI.hasSignedScratchOffsets())
6697 return true;
6698
6699 Register LHS = AddrMI->getOperand(1).getReg();
6700 Register RHS = AddrMI->getOperand(2).getReg();
6701
6702 if (AddrMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6703 std::optional<ValueAndVReg> RhsValReg =
6705 // If the immediate offset is negative and within certain range, the base
6706 // address cannot also be negative. If the base is also negative, the sum
6707 // would be either negative or much larger than the valid range of scratch
6708 // memory a thread can access.
6709 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6710 RhsValReg->Value.getSExtValue() > -0x40000000)
6711 return true;
6712 }
6713
6714 return VT->signBitIsZero(LHS);
6715}
6716
6717// Check address value in SGPR/VGPR are legal for flat scratch in the form
6718// of: SGPR + VGPR.
6719bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
6720 MachineInstr *AddrMI = getDefIgnoringCopies(Addr, *MRI);
6721
6722 if (isNoUnsignedWrap(AddrMI))
6723 return true;
6724
6725 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6726 // values.
6727 if (STI.hasSignedScratchOffsets())
6728 return true;
6729
6730 Register LHS = AddrMI->getOperand(1).getReg();
6731 Register RHS = AddrMI->getOperand(2).getReg();
6732 return VT->signBitIsZero(RHS) && VT->signBitIsZero(LHS);
6733}
6734
6735// Check address value in SGPR/VGPR are legal for flat scratch in the form
6736// of: SGPR + VGPR + Imm.
6737bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6738 Register Addr) const {
6739 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6740 // values.
6741 if (STI.hasSignedScratchOffsets())
6742 return true;
6743
6744 MachineInstr *AddrMI = getDefIgnoringCopies(Addr, *MRI);
6745 Register Base = AddrMI->getOperand(1).getReg();
6746 std::optional<DefinitionAndSourceRegister> BaseDef =
6748 std::optional<ValueAndVReg> RHSOffset =
6750 assert(RHSOffset);
6751
6752 // If the immediate offset is negative and within certain range, the base
6753 // address cannot also be negative. If the base is also negative, the sum
6754 // would be either negative or much larger than the valid range of scratch
6755 // memory a thread can access.
6756 if (isNoUnsignedWrap(BaseDef->MI) &&
6757 (isNoUnsignedWrap(AddrMI) ||
6758 (RHSOffset->Value.getSExtValue() < 0 &&
6759 RHSOffset->Value.getSExtValue() > -0x40000000)))
6760 return true;
6761
6762 Register LHS = BaseDef->MI->getOperand(1).getReg();
6763 Register RHS = BaseDef->MI->getOperand(2).getReg();
6764 return VT->signBitIsZero(RHS) && VT->signBitIsZero(LHS);
6765}
6766
6767bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
6768 unsigned ShAmtBits) const {
6769 assert(MI.getOpcode() == TargetOpcode::G_AND);
6770
6771 // The mask constant is an SGPR G_CONSTANT copied to VGPR; the matcher looks
6772 // through that COPY, so look through it here too.
6773 std::optional<ValueAndVReg> RHS =
6774 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), *MRI);
6775 if (!RHS)
6776 return false;
6777
6778 if (RHS->Value.countr_one() >= ShAmtBits)
6779 return true;
6780
6781 const APInt &LHSKnownZeros = VT->getKnownZeroes(MI.getOperand(1).getReg());
6782 return (LHSKnownZeros | RHS->Value).countr_one() >= ShAmtBits;
6783}
6784
6786AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6787 MachineOperand &Root) const {
6788 Register Reg = Root.getReg();
6789 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6790
6791 std::optional<DefinitionAndSourceRegister> Def =
6793 assert(Def && "this shouldn't be an optional result");
6794 Reg = Def->Reg;
6795
6796 if (Register WaveBase = getWaveAddress(Def->MI)) {
6797 return {{
6798 [=](MachineInstrBuilder &MIB) { // rsrc
6799 MIB.addReg(Info->getScratchRSrcReg());
6800 },
6801 [=](MachineInstrBuilder &MIB) { // soffset
6802 MIB.addReg(WaveBase);
6803 },
6804 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // offset
6805 }};
6806 }
6807
6808 int64_t Offset = 0;
6809
6810 // FIXME: Copy check is a hack
6812 if (mi_match(Reg, *MRI,
6813 m_GPtrAdd(m_Reg(BasePtr),
6815 if (!TII.isLegalMUBUFImmOffset(Offset))
6816 return {};
6817 MachineInstr *BasePtrDef = getDefIgnoringCopies(BasePtr, *MRI);
6818 Register WaveBase = getWaveAddress(BasePtrDef);
6819 if (!WaveBase)
6820 return {};
6821
6822 return {{
6823 [=](MachineInstrBuilder &MIB) { // rsrc
6824 MIB.addReg(Info->getScratchRSrcReg());
6825 },
6826 [=](MachineInstrBuilder &MIB) { // soffset
6827 MIB.addReg(WaveBase);
6828 },
6829 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } // offset
6830 }};
6831 }
6832
6833 if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) ||
6834 !TII.isLegalMUBUFImmOffset(Offset))
6835 return {};
6836
6837 return {{
6838 [=](MachineInstrBuilder &MIB) { // rsrc
6839 MIB.addReg(Info->getScratchRSrcReg());
6840 },
6841 [=](MachineInstrBuilder &MIB) { // soffset
6842 MIB.addImm(0);
6843 },
6844 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } // offset
6845 }};
6846}
6847
6848std::pair<Register, int64_t>
6849AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(
6850 MachineOperand &Root) const {
6851 int64_t ConstAddr = 0;
6852
6853 Register PtrBase;
6854 int64_t Offset;
6855 std::tie(PtrBase, Offset, std::ignore) =
6856 getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
6857
6858 if (Offset) {
6859 if (isDSOffsetLegal(PtrBase, Offset)) {
6860 // (add n0, c0)
6861 return std::pair(PtrBase, Offset);
6862 }
6863 } else if (mi_match(Root.getReg(), *MRI, m_GSub(m_Reg(), m_Reg()))) {
6864 // TODO
6865
6866 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) {
6867 // TODO
6868 }
6869
6870 return std::pair(Root.getReg(), 0);
6871}
6872
6874AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
6875 Register Reg;
6876 int64_t Offset;
6877 std::tie(Reg, Offset) = selectDS1Addr1OffsetImpl(Root);
6878 return {{
6879 [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
6880 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }
6881 }};
6882}
6883
6885AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
6886 return selectDSReadWrite2(Root, 4);
6887}
6888
6890AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
6891 return selectDSReadWrite2(Root, 8);
6892}
6893
6895AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
6896 unsigned Size) const {
6897 Register Reg;
6898 int64_t Offset;
6899 std::tie(Reg, Offset) = selectDSReadWrite2Impl(Root, Size);
6900 return {{
6901 [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
6902 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); },
6903 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset+1); }
6904 }};
6905}
6906
6907std::pair<Register, int64_t>
6908AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
6909 unsigned Size) const {
6910 int64_t ConstAddr = 0;
6911
6912 Register PtrBase;
6913 int64_t Offset;
6914 std::tie(PtrBase, Offset, std::ignore) =
6915 getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
6916
6917 if (Offset) {
6918 int64_t OffsetValue0 = Offset;
6919 int64_t OffsetValue1 = Offset + Size;
6920 if (isDSOffset2Legal(PtrBase, OffsetValue0, OffsetValue1, Size)) {
6921 // (add n0, c0)
6922 return std::pair(PtrBase, OffsetValue0 / Size);
6923 }
6924 } else if (mi_match(Root.getReg(), *MRI, m_GSub(m_Reg(), m_Reg()))) {
6925 // TODO
6926
6927 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) {
6928 // TODO
6929 }
6930
6931 return std::pair(Root.getReg(), 0);
6932}
6933
6934/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
6935/// the base value with the constant offset, and if the offset computation is
6936/// known to be inbounds. There may be intervening copies between \p Root and
6937/// the identified constant. Returns \p Root, 0, false if this does not match
6938/// the pattern.
6939std::tuple<Register, int64_t, bool>
6940AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6941 Register Root, const MachineRegisterInfo &MRI) const {
6942 MachineInstr *RootI = getDefIgnoringCopies(Root, MRI);
6943 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
6944 return {Root, 0, false};
6945
6946 MachineOperand &RHS = RootI->getOperand(2);
6947 std::optional<ValueAndVReg> MaybeOffset =
6949 if (!MaybeOffset)
6950 return {Root, 0, false};
6951 bool IsInBounds = RootI->getFlag(MachineInstr::MIFlag::InBounds);
6952 return {RootI->getOperand(1).getReg(), MaybeOffset->Value.getSExtValue(),
6953 IsInBounds};
6954}
6955
6957 MIB.addImm(0);
6958}
6959
6960/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
6961/// BasePtr is not valid, a null base pointer will be used.
6963 uint32_t FormatLo, uint32_t FormatHi,
6964 Register BasePtr) {
6965 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6966 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6967 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6968 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
6969
6970 B.buildInstr(AMDGPU::S_MOV_B32)
6971 .addDef(RSrc2)
6972 .addImm(FormatLo);
6973 B.buildInstr(AMDGPU::S_MOV_B32)
6974 .addDef(RSrc3)
6975 .addImm(FormatHi);
6976
6977 // Build the half of the subregister with the constants before building the
6978 // full 128-bit register. If we are building multiple resource descriptors,
6979 // this will allow CSEing of the 2-component register.
6980 B.buildInstr(AMDGPU::REG_SEQUENCE)
6981 .addDef(RSrcHi)
6982 .addReg(RSrc2)
6983 .addImm(AMDGPU::sub0)
6984 .addReg(RSrc3)
6985 .addImm(AMDGPU::sub1);
6986
6987 Register RSrcLo = BasePtr;
6988 if (!BasePtr) {
6989 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6990 B.buildInstr(AMDGPU::S_MOV_B64)
6991 .addDef(RSrcLo)
6992 .addImm(0);
6993 }
6994
6995 B.buildInstr(AMDGPU::REG_SEQUENCE)
6996 .addDef(RSrc)
6997 .addReg(RSrcLo)
6998 .addImm(AMDGPU::sub0_sub1)
6999 .addReg(RSrcHi)
7000 .addImm(AMDGPU::sub2_sub3);
7001
7002 return RSrc;
7003}
7004
7006 const SIInstrInfo &TII, Register BasePtr) {
7007 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
7008
7009 // FIXME: Why are half the "default" bits ignored based on the addressing
7010 // mode?
7011 return buildRSRC(B, MRI, 0, Hi_32(DefaultFormat), BasePtr);
7012}
7013
7015 const SIInstrInfo &TII, Register BasePtr) {
7016 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
7017
7018 // FIXME: Why are half the "default" bits ignored based on the addressing
7019 // mode?
7020 return buildRSRC(B, MRI, -1, Hi_32(DefaultFormat), BasePtr);
7021}
7022
7023AMDGPUInstructionSelector::MUBUFAddressData
7024AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
7025 MUBUFAddressData Data;
7026 Data.N0 = Src;
7027
7028 Register PtrBase;
7029 int64_t Offset;
7030
7031 std::tie(PtrBase, Offset, std::ignore) =
7032 getPtrBaseWithConstantOffset(Src, *MRI);
7033 if (isUInt<32>(Offset)) {
7034 Data.N0 = PtrBase;
7035 Data.Offset = Offset;
7036 }
7037
7038 if (MachineInstr *InputAdd
7039 = getOpcodeDef(TargetOpcode::G_PTR_ADD, Data.N0, *MRI)) {
7040 Data.N2 = InputAdd->getOperand(1).getReg();
7041 Data.N3 = InputAdd->getOperand(2).getReg();
7042
7043 // FIXME: Need to fix extra SGPR->VGPRcopies inserted
7044 // FIXME: Don't know this was defined by operand 0
7045 //
7046 // TODO: Remove this when we have copy folding optimizations after
7047 // RegBankSelect.
7048 Data.N2 = getDefIgnoringCopies(Data.N2, *MRI)->getOperand(0).getReg();
7049 Data.N3 = getDefIgnoringCopies(Data.N3, *MRI)->getOperand(0).getReg();
7050 }
7051
7052 return Data;
7053}
7054
7055/// Return if the addr64 mubuf mode should be used for the given address.
7056bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
7057 // (ptr_add N2, N3) -> addr64, or
7058 // (ptr_add (ptr_add N2, N3), C1) -> addr64
7059 if (Addr.N2)
7060 return true;
7061
7062 const RegisterBank *N0Bank = RBI.getRegBank(Addr.N0, *MRI, TRI);
7063 return N0Bank->getID() == AMDGPU::VGPRRegBankID;
7064}
7065
7066/// Split an immediate offset \p ImmOffset depending on whether it fits in the
7067/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
7068/// component.
7069void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
7070 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
7071 if (TII.isLegalMUBUFImmOffset(ImmOffset))
7072 return;
7073
7074 // Illegal offset, store it in soffset.
7075 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7076 B.buildInstr(AMDGPU::S_MOV_B32)
7077 .addDef(SOffset)
7078 .addImm(ImmOffset);
7079 ImmOffset = 0;
7080}
7081
7082bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
7083 MachineOperand &Root, Register &VAddr, Register &RSrcReg,
7084 Register &SOffset, int64_t &Offset) const {
7085 // FIXME: Predicates should stop this from reaching here.
7086 // addr64 bit was removed for volcanic islands.
7087 if (!STI.hasAddr64() || STI.useFlatForGlobal())
7088 return false;
7089
7090 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg());
7091 if (!shouldUseAddr64(AddrData))
7092 return false;
7093
7094 Register N0 = AddrData.N0;
7095 Register N2 = AddrData.N2;
7096 Register N3 = AddrData.N3;
7097 Offset = AddrData.Offset;
7098
7099 // Base pointer for the SRD.
7100 Register SRDPtr;
7101
7102 if (N2) {
7103 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7104 assert(N3);
7105 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7106 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
7107 // addr64, and construct the default resource from a 0 address.
7108 VAddr = N0;
7109 } else {
7110 SRDPtr = N3;
7111 VAddr = N2;
7112 }
7113 } else {
7114 // N2 is not divergent.
7115 SRDPtr = N2;
7116 VAddr = N3;
7117 }
7118 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7119 // Use the default null pointer in the resource
7120 VAddr = N0;
7121 } else {
7122 // N0 -> offset, or
7123 // (N0 + C1) -> offset
7124 SRDPtr = N0;
7125 }
7126
7127 MachineIRBuilder B(*Root.getParent());
7128 RSrcReg = buildAddr64RSrc(B, *MRI, TII, SRDPtr);
7129 splitIllegalMUBUFOffset(B, SOffset, Offset);
7130 return true;
7131}
7132
7133bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
7134 MachineOperand &Root, Register &RSrcReg, Register &SOffset,
7135 int64_t &Offset) const {
7136
7137 // FIXME: Pattern should not reach here.
7138 if (STI.useFlatForGlobal())
7139 return false;
7140
7141 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg());
7142 if (shouldUseAddr64(AddrData))
7143 return false;
7144
7145 // N0 -> offset, or
7146 // (N0 + C1) -> offset
7147 Register SRDPtr = AddrData.N0;
7148 Offset = AddrData.Offset;
7149
7150 // TODO: Look through extensions for 32-bit soffset.
7151 MachineIRBuilder B(*Root.getParent());
7152
7153 RSrcReg = buildOffsetSrc(B, *MRI, TII, SRDPtr);
7154 splitIllegalMUBUFOffset(B, SOffset, Offset);
7155 return true;
7156}
7157
7159AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
7160 Register VAddr;
7161 Register RSrcReg;
7162 Register SOffset;
7163 int64_t Offset = 0;
7164
7165 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
7166 return {};
7167
7168 // FIXME: Use defaulted operands for trailing 0s and remove from the complex
7169 // pattern.
7170 return {{
7171 [=](MachineInstrBuilder &MIB) { // rsrc
7172 MIB.addReg(RSrcReg);
7173 },
7174 [=](MachineInstrBuilder &MIB) { // vaddr
7175 MIB.addReg(VAddr);
7176 },
7177 [=](MachineInstrBuilder &MIB) { // soffset
7178 if (SOffset)
7179 MIB.addReg(SOffset);
7180 else if (STI.hasRestrictedSOffset())
7181 MIB.addReg(AMDGPU::SGPR_NULL);
7182 else
7183 MIB.addImm(0);
7184 },
7185 [=](MachineInstrBuilder &MIB) { // offset
7186 MIB.addImm(Offset);
7187 },
7188 addZeroImm, // cpol
7189 addZeroImm, // tfe
7190 addZeroImm // swz
7191 }};
7192}
7193
7195AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
7196 Register RSrcReg;
7197 Register SOffset;
7198 int64_t Offset = 0;
7199
7200 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
7201 return {};
7202
7203 return {{
7204 [=](MachineInstrBuilder &MIB) { // rsrc
7205 MIB.addReg(RSrcReg);
7206 },
7207 [=](MachineInstrBuilder &MIB) { // soffset
7208 if (SOffset)
7209 MIB.addReg(SOffset);
7210 else if (STI.hasRestrictedSOffset())
7211 MIB.addReg(AMDGPU::SGPR_NULL);
7212 else
7213 MIB.addImm(0);
7214 },
7215 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }, // offset
7216 addZeroImm, // cpol
7217 addZeroImm, // tfe
7218 addZeroImm, // swz
7219 }};
7220}
7221
7223AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const {
7224
7225 Register SOffset = Root.getReg();
7226
7227 if (STI.hasRestrictedSOffset() && mi_match(SOffset, *MRI, m_ZeroInt()))
7228 SOffset = AMDGPU::SGPR_NULL;
7229
7230 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(SOffset); }}};
7231}
7232
7233/// Get an immediate that must be 32-bits, and treated as zero extended.
7234static std::optional<uint64_t>
7236 // getIConstantVRegVal sexts any values, so see if that matters.
7237 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(Reg, MRI);
7238 if (!OffsetVal || !isInt<32>(*OffsetVal))
7239 return std::nullopt;
7240 return Lo_32(*OffsetVal);
7241}
7242
7244AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
7245 std::optional<uint64_t> OffsetVal =
7246 Root.isImm() ? Root.getImm() : getConstantZext32Val(Root.getReg(), *MRI);
7247 if (!OffsetVal)
7248 return {};
7249
7250 std::optional<int64_t> EncodedImm =
7251 AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal, true);
7252 if (!EncodedImm)
7253 return {};
7254
7255 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); } }};
7256}
7257
7259AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
7260 assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
7261
7262 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI);
7263 if (!OffsetVal)
7264 return {};
7265
7266 std::optional<int64_t> EncodedImm =
7268 if (!EncodedImm)
7269 return {};
7270
7271 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); } }};
7272}
7273
7275AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
7276 // Match the (soffset + offset) pair as a 32-bit register base and
7277 // an immediate offset.
7278 Register SOffset;
7279 unsigned Offset;
7280 std::tie(SOffset, Offset) = AMDGPU::getBaseWithConstantOffset(
7281 *MRI, Root.getReg(), VT, /*CheckNUW*/ true);
7282 if (!SOffset)
7283 return std::nullopt;
7284
7285 std::optional<int64_t> EncodedOffset =
7286 AMDGPU::getSMRDEncodedOffset(STI, Offset, /* IsBuffer */ true);
7287 if (!EncodedOffset)
7288 return std::nullopt;
7289
7290 assert(MRI->getType(SOffset).getSizeInBits() == 32);
7291 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(SOffset); },
7292 [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedOffset); }}};
7293}
7294
7295std::pair<Register, unsigned>
7296AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
7297 bool &Matched) const {
7298 Matched = false;
7299
7300 Register Src;
7301 unsigned Mods;
7302 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg());
7303
7304 if (mi_match(Src, *MRI, m_GFPExt(m_Reg(Src)))) {
7305 assert(MRI->getType(Src) == LLT::scalar(16));
7306
7307 // Only change Src if src modifier could be gained. In such cases new Src
7308 // could be sgpr but this does not violate constant bus restriction for
7309 // instruction that is being selected.
7310 Src = stripBitCast(Src, *MRI);
7311
7312 const auto CheckAbsNeg = [&]() {
7313 // Be careful about folding modifiers if we already have an abs. fneg is
7314 // applied last, so we don't want to apply an earlier fneg.
7315 if ((Mods & SISrcMods::ABS) == 0) {
7316 unsigned ModsTmp;
7317 std::tie(Src, ModsTmp) = selectVOP3ModsImpl(Src);
7318
7319 if ((ModsTmp & SISrcMods::NEG) != 0)
7320 Mods ^= SISrcMods::NEG;
7321
7322 if ((ModsTmp & SISrcMods::ABS) != 0)
7323 Mods |= SISrcMods::ABS;
7324 }
7325 };
7326
7327 CheckAbsNeg();
7328
7329 // op_sel/op_sel_hi decide the source type and source.
7330 // If the source's op_sel_hi is set, it indicates to do a conversion from
7331 // fp16. If the sources's op_sel is set, it picks the high half of the
7332 // source register.
7333
7334 Mods |= SISrcMods::OP_SEL_1;
7335
7336 // The instruction reads a 32-bit source and selects a half of it, so look
7337 // for the 32-bit register the 16-bit value is a half of.
7338 if (isExtractHiElt(*MRI, Src, Src)) {
7339 // Src is now the 32-bit source and op_sel picks its high half.
7340 Mods |= SISrcMods::OP_SEL_0;
7341 CheckAbsNeg();
7342 } else if (isExtractLoElt(*MRI, Src, Src)) {
7343 // op_sel already picks the low half, so the 32-bit source can be used
7344 // directly. Unlike the high half, only an fneg/fabs that acts on each
7345 // 16-bit element can be folded here: one that acts on the 32-bit value
7346 // touches bit 31 and leaves the low half alone.
7347 if (MRI->getType(Src).isFixedVector(2, 16))
7348 CheckAbsNeg();
7349 }
7350 // Otherwise Src is genuinely 16 bits wide and widenSrcIfVGPR16 widens it
7351 // when the operand is rendered.
7352
7353 Matched = true;
7354 }
7355
7356 return {Src, Mods};
7357}
7358
7360AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7361 MachineOperand &Root) const {
7362 Register Src;
7363 unsigned Mods;
7364 bool Matched;
7365 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7366 if (!Matched)
7367 return {};
7368
7369 return {{
7370 [=](MachineInstrBuilder &MIB) { MIB.addReg(widenSrcIfVGPR16(Src, MIB)); },
7371 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
7372 }};
7373}
7374
7376AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
7377 Register Src;
7378 unsigned Mods;
7379 bool Matched;
7380 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7381
7382 return {{
7383 [=](MachineInstrBuilder &MIB) { MIB.addReg(widenSrcIfVGPR16(Src, MIB)); },
7384 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods
7385 }};
7386}
7387
7389AMDGPUInstructionSelector::selectVOP3PMadMixModsExtNeg(
7390 MachineOperand &Root) const {
7391 Register Src;
7392 unsigned Mods;
7393 bool Matched;
7394 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7395 if (!Matched)
7396 return {};
7397
7398 return {{
7399 [=](MachineInstrBuilder &MIB) { MIB.addReg(widenSrcIfVGPR16(Src, MIB)); },
7400 [=](MachineInstrBuilder &MIB) {
7401 MIB.addImm(Mods ^ SISrcMods::NEG);
7402 } // src_mods
7403 }};
7404}
7405
7407AMDGPUInstructionSelector::selectVOP3PMadMixModsNeg(
7408 MachineOperand &Root) const {
7409 Register Src;
7410 unsigned Mods;
7411 bool Matched;
7412 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7413
7414 return {{
7415 [=](MachineInstrBuilder &MIB) { MIB.addReg(widenSrcIfVGPR16(Src, MIB)); },
7416 [=](MachineInstrBuilder &MIB) {
7417 MIB.addImm(Mods ^ SISrcMods::NEG);
7418 } // src_mods
7419 }};
7420}
7421
7422bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7423 MachineInstr &I, Intrinsic::ID IntrID) const {
7424 MachineBasicBlock *MBB = I.getParent();
7425 const DebugLoc &DL = I.getDebugLoc();
7426 Register CCReg = I.getOperand(0).getReg();
7427
7428 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7429 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_CMP_EQ_U32)).addImm(0).addImm(0);
7430
7431 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7432 .addImm(I.getOperand(2).getImm());
7433
7434 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), CCReg).addReg(AMDGPU::SCC);
7435
7436 I.eraseFromParent();
7437 return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass,
7438 *MRI);
7439}
7440
7441bool AMDGPUInstructionSelector::selectSGetBarrierState(
7442 MachineInstr &I, Intrinsic::ID IntrID) const {
7443 MachineBasicBlock *MBB = I.getParent();
7444 const DebugLoc &DL = I.getDebugLoc();
7445 const MachineOperand &BarOp = I.getOperand(2);
7446 std::optional<int64_t> BarValImm =
7447 getIConstantVRegSExtVal(BarOp.getReg(), *MRI);
7448
7449 if (!BarValImm) {
7450 auto CopyMIB = BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
7451 .addReg(BarOp.getReg());
7452 constrainSelectedInstRegOperands(*CopyMIB, TII, TRI, RBI);
7453 }
7454 MachineInstrBuilder MIB;
7455 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7456 : AMDGPU::S_GET_BARRIER_STATE_M0;
7457 MIB = BuildMI(*MBB, &I, DL, TII.get(Opc));
7458
7459 auto DstReg = I.getOperand(0).getReg();
7460 const TargetRegisterClass *DstRC =
7461 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
7462 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
7463 return false;
7464 MIB.addDef(DstReg);
7465 if (BarValImm) {
7466 MIB.addImm(*BarValImm);
7467 }
7468 I.eraseFromParent();
7469 return true;
7470}
7471
7472/// Return the barrier ID for \p BarOp if it's statically known, mapping poison
7473/// to 0 and masking off the high bits, since they're ignored as reserved.
7474static std::optional<uint32_t>
7476 const MachineRegisterInfo &MRI) {
7477 Register BarReg = BarOp.getReg();
7478 if (std::optional<int64_t> BarValImm = getIConstantVRegSExtVal(BarReg, MRI))
7479 return *BarValImm & 0x3F;
7480 if (getOpcodeDef(AMDGPU::G_IMPLICIT_DEF, BarReg, MRI))
7481 return 0;
7482 return std::nullopt;
7483}
7484
7485unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
7486 if (HasInlineConst) {
7487 switch (IntrID) {
7488 default:
7489 llvm_unreachable("not a named barrier op");
7490 case Intrinsic::amdgcn_s_barrier_join:
7491 return AMDGPU::S_BARRIER_JOIN_IMM;
7492 case Intrinsic::amdgcn_s_wakeup_barrier:
7493 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7494 case Intrinsic::amdgcn_s_get_named_barrier_state:
7495 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7496 };
7497 } else {
7498 switch (IntrID) {
7499 default:
7500 llvm_unreachable("not a named barrier op");
7501 case Intrinsic::amdgcn_s_barrier_join:
7502 return AMDGPU::S_BARRIER_JOIN_M0;
7503 case Intrinsic::amdgcn_s_wakeup_barrier:
7504 return AMDGPU::S_WAKEUP_BARRIER_M0;
7505 case Intrinsic::amdgcn_s_get_named_barrier_state:
7506 return AMDGPU::S_GET_BARRIER_STATE_M0;
7507 };
7508 }
7509}
7510
7511bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7512 MachineInstr &I, Intrinsic::ID IntrID) const {
7513 MachineBasicBlock *MBB = I.getParent();
7514 const DebugLoc &DL = I.getDebugLoc();
7515 const MachineOperand &BarOp = I.getOperand(1);
7516 const MachineOperand &CntOp = I.getOperand(2);
7517
7518 // A member count of 0 means "keep existing member count". That plus a known
7519 // constant value for the barrier ID lets us use the immarg form.
7520 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7521 std::optional<int64_t> CntImm =
7522 getIConstantVRegSExtVal(CntOp.getReg(), *MRI);
7523 if (CntImm && *CntImm == 0) {
7524 if (auto BarID = getConstantNamedBarrierNumber(BarOp, *MRI)) {
7525 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_IMM))
7526 .addImm(*BarID);
7527 I.eraseFromParent();
7528 return true;
7529 }
7530 }
7531 }
7532
7533 // BarID = BarOp & 0x3F
7534 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7535 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_AND_B32), TmpReg1)
7536 .add(BarOp)
7537 .addImm(0x3F)
7538 .setOperandDead(3); // Dead scc
7539
7540 // MO = ((CntOp & CntMask) << shAmt) | BarID
7541 unsigned CntMask =
7543 Register TmpReg2 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7544 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_AND_B32), TmpReg2)
7545 .add(CntOp)
7546 .addImm(CntMask)
7547 .setOperandDead(3); // Dead scc
7548
7549 Register TmpReg3 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7550 constexpr unsigned ShAmt = 16;
7551 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg3)
7552 .addReg(TmpReg2)
7553 .addImm(ShAmt)
7554 .setOperandDead(3); // Dead scc
7555
7556 Register TmpReg4 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7557 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_OR_B32), TmpReg4)
7558 .addReg(TmpReg1)
7559 .addReg(TmpReg3)
7560 .setOperandDead(3); // Dead scc;
7561
7562 auto CopyMIB =
7563 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0).addReg(TmpReg4);
7564 constrainSelectedInstRegOperands(*CopyMIB, TII, TRI, RBI);
7565
7566 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7567 ? AMDGPU::S_BARRIER_INIT_M0
7568 : AMDGPU::S_BARRIER_SIGNAL_M0;
7569 MachineInstrBuilder MIB;
7570 MIB = BuildMI(*MBB, &I, DL, TII.get(Opc));
7571
7572 I.eraseFromParent();
7573 return true;
7574}
7575
7576bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7577 MachineInstr &I, Intrinsic::ID IntrID) const {
7578 MachineBasicBlock *MBB = I.getParent();
7579 const DebugLoc &DL = I.getDebugLoc();
7580 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7581 ? I.getOperand(2)
7582 : I.getOperand(1);
7583 std::optional<uint32_t> BarValImm =
7584 getConstantNamedBarrierNumber(BarOp, *MRI);
7585
7586 if (!BarValImm) {
7587 // BarID = BarOp & 0x3F
7588 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7589 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_AND_B32), TmpReg1)
7590 .addReg(BarOp.getReg())
7591 .addImm(0x3F)
7592 .setOperandDead(3); // Dead scc;
7593
7594 auto CopyMIB = BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
7595 .addReg(TmpReg1);
7596 constrainSelectedInstRegOperands(*CopyMIB, TII, TRI, RBI);
7597 }
7598
7599 MachineInstrBuilder MIB;
7600 unsigned Opc = getNamedBarrierOp(BarValImm.has_value(), IntrID);
7601 MIB = BuildMI(*MBB, &I, DL, TII.get(Opc));
7602
7603 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7604 auto DstReg = I.getOperand(0).getReg();
7605 const TargetRegisterClass *DstRC =
7606 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
7607 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
7608 return false;
7609 MIB.addDef(DstReg);
7610 }
7611
7612 if (BarValImm)
7613 MIB.addImm(*BarValImm);
7614
7615 I.eraseFromParent();
7616 return true;
7617}
7618
7619void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
7620 const MachineInstr &MI,
7621 int OpIdx) const {
7622 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7623 "Expected G_CONSTANT");
7624 MIB.addImm(-MI.getOperand(1).getCImm()->getSExtValue());
7625}
7626
7627void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
7628 const MachineInstr &MI,
7629 int OpIdx) const {
7630 const MachineOperand &Op = MI.getOperand(1);
7631 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7632 MIB.addImm(Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7633}
7634
7635void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7636 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7637 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7638 "Expected G_CONSTANT");
7639 MIB.addImm(MI.getOperand(1).getCImm()->getValue().countTrailingOnes());
7640}
7641
7642/// This only really exists to satisfy DAG type checking machinery, so is a
7643/// no-op here.
7644void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
7645 const MachineInstr &MI,
7646 int OpIdx) const {
7647 const MachineOperand &Op = MI.getOperand(OpIdx);
7648 int64_t Imm;
7649 if (Op.isReg() && mi_match(Op.getReg(), *MRI, m_ICst(Imm)))
7650 MIB.addImm(Imm);
7651 else
7652 MIB.addImm(Op.getImm());
7653}
7654
7655void AMDGPUInstructionSelector::renderPrefetchLen(MachineInstrBuilder &MIB,
7656 const MachineInstr &MI,
7657 int OpIdx) const {
7658 Register Len = MI.getOperand(OpIdx).getReg();
7659 MIB.addImm(*getIConstantVRegSExtVal(Len, *MRI) & 0x1f);
7660}
7661
7662void AMDGPUInstructionSelector::renderZextBoolTImm(MachineInstrBuilder &MIB,
7663 const MachineInstr &MI,
7664 int OpIdx) const {
7665 MIB.addImm(MI.getOperand(OpIdx).getImm() != 0);
7666}
7667
7668void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
7669 const MachineInstr &MI,
7670 int OpIdx) const {
7671 assert(OpIdx >= 0 && "expected to match an immediate operand");
7672 MIB.addImm(MI.getOperand(OpIdx).getImm() ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7673}
7674
7675void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7676 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7677 assert(OpIdx >= 0 && "expected to match an immediate operand");
7678 MIB.addImm(
7679 (MI.getOperand(OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7680}
7681
7682void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7683 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7684 assert(OpIdx >= 0 && "expected to match an immediate operand");
7685 MIB.addImm((MI.getOperand(OpIdx).getImm() & 0x1)
7687 : (int64_t)SISrcMods::DST_OP_SEL);
7688}
7689
7690void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7691 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7692 assert(OpIdx >= 0 && "expected to match an immediate operand");
7693 MIB.addImm(
7694 (MI.getOperand(OpIdx).getImm() & 0x2) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7695}
7696
7697void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7698 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7699 assert(OpIdx >= 0 && "expected to match an immediate operand");
7700 MIB.addImm((MI.getOperand(OpIdx).getImm() & 0x2)
7701 ? (int64_t)(SISrcMods::OP_SEL_0)
7702 : 0);
7703}
7704
7705void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7706 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7707 assert(OpIdx >= 0 && "expected to match an immediate operand");
7708 MIB.addImm(MI.getOperand(OpIdx).getImm() ? (int64_t)(SISrcMods::DST_OP_SEL)
7709 : 0);
7710}
7711
7712void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7713 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7714 assert(OpIdx >= 0 && "expected to match an immediate operand");
7715 MIB.addImm(MI.getOperand(OpIdx).getImm() ? (int64_t)(SISrcMods::OP_SEL_0)
7716 : 0);
7717}
7718
7719void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7720 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7721 assert(OpIdx >= 0 && "expected to match an immediate operand");
7722 MIB.addImm(
7723 (MI.getOperand(OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7724}
7725
7726void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7727 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7728 assert(OpIdx >= 0 && "expected to match an immediate operand");
7729 MIB.addImm((MI.getOperand(OpIdx).getImm() & 0x2)
7730 ? (int64_t)SISrcMods::DST_OP_SEL
7731 : 0);
7732}
7733
7734void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
7735 const MachineInstr &MI,
7736 int OpIdx) const {
7737 assert(OpIdx >= 0 && "expected to match an immediate operand");
7738 MIB.addImm(MI.getOperand(OpIdx).getImm() &
7741}
7742
7743void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
7744 const MachineInstr &MI,
7745 int OpIdx) const {
7746 assert(OpIdx >= 0 && "expected to match an immediate operand");
7747 const bool Swizzle = MI.getOperand(OpIdx).getImm() &
7750 MIB.addImm(Swizzle);
7751}
7752
7753void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7754 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7755 assert(OpIdx >= 0 && "expected to match an immediate operand");
7756 const uint32_t Cpol = MI.getOperand(OpIdx).getImm() &
7759 MIB.addImm(Cpol | AMDGPU::CPol::GLC);
7760}
7761
7762void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
7763 const MachineInstr &MI,
7764 int OpIdx) const {
7765 const APFloat &APF = MI.getOperand(1).getFPImm()->getValueAPF();
7766 int ExpVal = APF.getExactLog2Abs();
7767 assert(ExpVal != INT_MIN);
7768 MIB.addImm(ExpVal);
7769}
7770
7771void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
7772 const MachineInstr &MI,
7773 int OpIdx) const {
7774 // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
7775 // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
7776 // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
7777 // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
7778 MIB.addImm((MI.getOperand(OpIdx).getImm() + 3) % 4);
7779}
7780
7781void AMDGPUInstructionSelector::renderVOP3PModsNeg(MachineInstrBuilder &MIB,
7782 const MachineInstr &MI,
7783 int OpIdx) const {
7784 unsigned Mods = SISrcMods::OP_SEL_1;
7785 if (MI.getOperand(OpIdx).getImm())
7786 Mods ^= SISrcMods::NEG;
7787 MIB.addImm((int64_t)Mods);
7788}
7789
7790void AMDGPUInstructionSelector::renderVOP3PModsNegs(MachineInstrBuilder &MIB,
7791 const MachineInstr &MI,
7792 int OpIdx) const {
7793 unsigned Mods = SISrcMods::OP_SEL_1;
7794 if (MI.getOperand(OpIdx).getImm())
7796 MIB.addImm((int64_t)Mods);
7797}
7798
7799void AMDGPUInstructionSelector::renderVOP3PModsNegAbs(MachineInstrBuilder &MIB,
7800 const MachineInstr &MI,
7801 int OpIdx) const {
7802 unsigned Val = static_cast<unsigned>(MI.getOperand(OpIdx).getImm());
7803 unsigned Mods = SISrcMods::OP_SEL_1; // default: none
7804 if (Val == 1) // neg
7805 Mods ^= SISrcMods::NEG;
7806 if (Val == 2) // abs
7807 Mods ^= SISrcMods::ABS;
7808 if (Val == 3) // neg and abs
7809 Mods ^= (SISrcMods::NEG | SISrcMods::ABS);
7810 MIB.addImm((int64_t)Mods);
7811}
7812
7813void AMDGPUInstructionSelector::renderPrefetchLoc(MachineInstrBuilder &MIB,
7814 const MachineInstr &MI,
7815 int OpIdx) const {
7816 uint32_t V = MI.getOperand(2).getImm() & AMDGPU::CPol::SCOPE_MASK;
7818 if (!Subtarget->hasSafeCUPrefetch())
7819 V = std::max(V, (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe
7820 MIB.addImm(V);
7821}
7822
7823/// Convert from 2-bit value to enum values used for op_sel* source modifiers.
7824void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7825 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7826 int64_t Val = MI.getOperand(OpIdx).getImm();
7827 unsigned New = 0;
7828 if (Val & 0x1)
7830 if (Val & 0x2)
7832 MIB.addImm(New);
7833}
7834
7835bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const {
7836 return TII.isInlineConstant(Imm);
7837}
7838
7839bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
7840 return TII.isInlineConstant(Imm);
7841}
MachineInstrBuilder MachineInstrBuilder & DefMI
static unsigned getIntrinsicID(const SDNode *N)
SDValue Pred
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test if the MI is shift left with half bits, such as reg0:2n =G_SHL reg1:2n, CONST(n)
static bool isNoUnsignedWrap(MachineInstr *Addr)
static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, const SIInstrInfo &TII, Register BasePtr)
unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID)
static Register getLegalRegBank(Register NewReg, Register RootReg, MachineInstr &Use, const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const SIInstrInfo &TII)
static bool checkRB(Register Reg, unsigned int RBNo, const AMDGPURegisterBankInfo &RBI, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI)
static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods)
static bool isTruncHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test if the MI is truncating to half, such as reg0:n = G_TRUNC reg1:2n
static Register getWaveAddress(const MachineInstr *Def)
static int getV_CMPOpcode(CmpInst::Predicate P, uint64_t Size, const GCNSubtarget &ST)
static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In, Register &Out)
static std::pair< unsigned, uint8_t > BitOp3_Op(Register R, SmallVectorImpl< Register > &Src, const MachineRegisterInfo &MRI)
static TypeClass isVectorOfTwoOrScalar(Register Reg, const MachineRegisterInfo &MRI)
static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI, MachineBasicBlock *MBB)
static bool isExtractLoElt(MachineRegisterInfo &MRI, Register In, Register &Out)
static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE, bool &IsTexFail)
static void addZeroImm(MachineInstrBuilder &MIB)
static unsigned gwsIntrinToOpcode(unsigned IntrID)
static bool isConstant(const MachineInstr &MI)
static bool isSameBitWidth(Register Reg1, Register Reg2, const MachineRegisterInfo &MRI)
static Register buildRegSequence(SmallVectorImpl< Register > &Elts, MachineInstr *InsertPt, MachineRegisterInfo &MRI)
static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI, uint32_t FormatLo, uint32_t FormatHi, Register BasePtr)
Return a resource descriptor for use with an arbitrary 64-bit pointer.
static bool isAsyncLDSDMA(Intrinsic::ID Intr)
static std::optional< uint32_t > getConstantNamedBarrierNumber(const MachineOperand &BarOp, const MachineRegisterInfo &MRI)
Return the barrier ID for BarOp if it's statically known, mapping poison to 0 and masking off the hig...
static std::pair< Register, unsigned > computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, Register IdxReg, unsigned EltSize, GISelValueTracking &ValueTracking)
Return the register to use for the index value, and the subregister to use for the indirectly accesse...
static bool shouldUseAndMask(uint64_t Size, unsigned &Mask)
static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64)
static std::pair< Register, SrcStatus > getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO, int MaxDepth=3)
static Register stripCopy(Register Reg, MachineRegisterInfo &MRI)
static std::optional< std::pair< Register, SrcStatus > > calcNextStatus(std::pair< Register, SrcStatus > Curr, const MachineRegisterInfo &MRI)
static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI)
static std::optional< uint64_t > getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI)
Get an immediate that must be 32-bits, and treated as zero extended.
static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg, Register RootReg, const SIInstrInfo &TII, const MachineRegisterInfo &MRI)
static SmallVector< std::pair< Register, SrcStatus > > getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO, int MaxDepth=3)
static bool isUnmergeHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test function, if the MI is reg0:n, reg1:n = G_UNMERGE_VALUES reg2:2n
static SrcStatus getNegStatus(Register Reg, SrcStatus S, const MachineRegisterInfo &MRI)
static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI)
static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, const SIInstrInfo &TII, Register BasePtr)
static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test if the MI is logic shift right with half bits, such as reg0:2n =G_LSHR reg1:2n,...
static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods, SmallVectorImpl< Register > &Elts, Register &Src, MachineInstr *InsertPt, MachineRegisterInfo &MRI)
This file declares the targeting of the InstructionSelector class for AMDGPU.
constexpr LLT S1
constexpr LLT S32
AMDGPU Register Bank Select
This file declares the targeting of the RegisterBankInfo class for AMDGPU.
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool isAllZeros(StringRef Arr)
Return true if the array is empty or all zeros.
dxil translate DXIL Translate Metadata
Provides analysis for querying information about KnownBits during GISel passes.
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define P(N)
static std::vector< std::pair< int, unsigned > > Swizzle(std::vector< std::pair< int, unsigned > > Src, R600InstrInfo::BankSwizzle Swz)
MachineRegisterInfo & MRI
#define LLVM_DEBUG(...)
Definition Debug.h:119
Value * RHS
Value * LHS
This is used to control valid status that current MI supports.
bool checkOptions(SrcStatus Stat) const
SearchOptions(Register Reg, const MachineRegisterInfo &MRI)
AMDGPUInstructionSelector(const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
static const char * getName()
bool select(MachineInstr &I) override
Select the (possibly generic) instruction I to only use target-specific opcodes.
void setupMF(MachineFunction &MF, GISelValueTracking *VT, CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) override
Setup per-MF executor state.
LLVM_READONLY int getExactLog2Abs() const
Definition APFloat.h:1654
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition InstrTypes.h:757
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition InstrTypes.h:742
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:174
DILocation * get() const
Get the underlying DILocation.
Definition DebugLoc.h:226
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
void checkSubtargetFeatures(const Function &F) const
Diagnose inconsistent subtarget features before attempting to codegen function F.
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
virtual void setupMF(MachineFunction &mf, GISelValueTracking *vt, CodeGenCoverage *covinfo=nullptr, ProfileSummaryInfo *psi=nullptr, BlockFrequencyInfo *bfi=nullptr)
Setup per-MF executor state.
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents a G_UNMERGE_VALUES.
unsigned getNumDefs() const
Returns the number of def registers.
Register getSourceReg() const
Get the unmerge source register.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
static LLT integer(unsigned SizeInBits)
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool hasValue() const
TypeSize getValue() const
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
bool hasSuperClassEq(const MCRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
unsigned getID() const
getID() - Return the register class ID number.
bool hasSubClassEq(const MCRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setReturnAddressIsTaken(bool s)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Helper class to build MachineInstr.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
LocationSize getSize() const
Return the size in bytes of the memory reference.
unsigned getAddrSpace() const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
const Value * getValue() const
Return the base address of the memory access.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
const ConstantInt * getCImm() const
void setImm(int64_t immVal)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
bool isEarlyClobber() const
Register getReg() const
getReg - Returns the register number.
bool isInternalRead() const
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Analysis providing profile information.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
const SDValue & getOperand(unsigned i) const
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
static unsigned getDSShaderTypeValue(const MachineFunction &MF)
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
static bool isGenericOpcode(unsigned Opc)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI Value(Type *Ty, unsigned scid)
Definition Value.cpp:49
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
LLVM_READONLY const MIMGG16MappingInfo * getMIMGG16MappingInfo(unsigned G)
std::optional< int64_t > getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, int64_t ByteOffset)
bool isGFX12Plus(const MCSubtargetInfo &STI)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc, bool IndexedSamp)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST)
LLVM_READONLY int32_t getGlobalSaddrOp(uint32_t Opcode)
bool isGFX13Plus(const MCSubtargetInfo &STI)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isGFX10Plus(const MCSubtargetInfo &STI)
std::optional< int64_t > getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset, bool IsBuffer, bool HasSOffset)
unsigned getMIROperandIdx(const MachineInstr &MI, unsigned Arg)
Convert IR argument index Arg of the intrinsic called by MI, a G_INTRINSIC* instruction,...
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
unsigned getBarrierMemberCountBitMask(const IsaVersion &Version)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
Intrinsic::ID getIntrinsicID(const MachineInstr &I)
Return the intrinsic ID for opcodes with the G_AMDGPU_INTRIN_ prefix.
std::pair< Register, unsigned > getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg, GISelValueTracking *ValueTracking=nullptr, bool CheckNUW=false)
Returns base register and constant offset.
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
IndexMode
ARM Index Modes.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue)
Matches a constant equal to RequestedValue.
GInstrBind< GBuildVector > m_GBuildVector(GBuildVector *&Inst)
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
UnaryOp_match< SrcTy, TargetOpcode::COPY > m_Copy(SrcTy &&Src)
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_XOR, true > m_GXor(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_SEXT > m_GSExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_FPEXT > m_GFPExt(const SrcTy &Src)
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
ConstantMatch< APInt > m_ICst(APInt &Cst)
SpecificConstantMatch m_AllOnesInt()
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
ImplicitDefMatch m_GImplicitDef()
GInstrBind< GConcatVectors > m_GConcatVectors(GConcatVectors *&Inst)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
GInstrBind< GUnmerge > m_GUnmerge(GUnmerge *&Inst)
Instruction binders for ops with no operand-form matcher (constant-immediate or variadic-source ops).
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
SpecificRegisterMatch m_SpecificReg(Register RequestedReg)
Matches a register only if it is equal to RequestedReg.
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
GFrameIndexMatch m_GFrameIndex(int &FI)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_BITCAST > m_GBitcast(const SrcTy &Src)
bind_ty< MachineInstr * > m_MInstr(MachineInstr *&MI)
UnaryOp_match< SrcTy, TargetOpcode::G_FNEG > m_GFNeg(const SrcTy &Src)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
UnaryOp_match< SrcTy, TargetOpcode::G_FABS > m_GFabs(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ANYEXT > m_GAnyExt(const SrcTy &Src)
ShuffleVectorMatch< Src1Ty, Src2Ty > m_GShuffleVector(const Src1Ty &Src1, const Src2Ty &Src2, ArrayRef< int > &Mask)
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, TargetOpcode::G_MUL, true > m_GMul(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
constexpr MCPhysReg SP
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
Definition Utils.cpp:896
@ Offset
Definition DWP.cpp:577
LLVM_ABI bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
Definition Utils.cpp:1593
LLVM_ABI Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
Definition Utils.cpp:64
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition Utils.cpp:679
PointerUnion< const TargetRegisterClass *, const RegisterBank * > RegClassOrRegBank
Convenient type to represent either a register class or a register bank.
LLVM_ABI const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
Definition Utils.cpp:487
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
@ Known
Known to have no common set bits.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
Definition MathExtras.h:345
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition Utils.cpp:163
@ Load
The value being inserted comes from a load (InsertElement only).
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition Utils.cpp:520
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:332
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition Utils.cpp:321
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI std::optional< ValueAndVReg > getAnyConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true, bool LookThroughAnyExt=false)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT or G_FCONST...
Definition Utils.cpp:446
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
LLVM_ABI std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Definition Utils.cpp:495
LLVM_ABI Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
Definition Utils.cpp:527
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
constexpr RegState getUndefRegState(bool B)
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
The argument layout of a buffer intrinsic.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:315
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
Definition KnownBits.h:361
int64_t Offset
Offset - This is an offset from the base Value*.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.