28#include "llvm/IR/IntrinsicsAMDGPU.h"
31#define DEBUG_TYPE "amdgpu-isel"
36#define GET_GLOBALISEL_IMPL
37#define AMDGPUSubtarget GCNSubtarget
38#include "AMDGPUGenGlobalISel.inc"
39#undef GET_GLOBALISEL_IMPL
44 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
46#include
"AMDGPUGenGlobalISel.inc"
49#include
"AMDGPUGenGlobalISel.inc"
61 MRI = &
MF.getRegInfo();
69 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
70 ? Def->getOperand(1).getReg()
80 auto &RegClassOrBank = MRI.getRegClassOrRegBank(
Reg);
84 const LLT Ty = MRI.getType(
Reg);
93 return RB->
getID() == AMDGPU::VCCRegBankID;
96bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(
MachineInstr &
MI,
97 unsigned NewOpc)
const {
98 MI.setDesc(TII.get(NewOpc));
110 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
112 TRI.getConstrainedRegClassForReg(SrcReg, *MRI);
113 if (!DstRC || DstRC != SrcRC)
116 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
117 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
119 const MCInstrDesc &MCID =
MI.getDesc();
121 MI.getOperand(0).setIsEarlyClobber(
true);
126bool AMDGPUInstructionSelector::selectCOPY(
MachineInstr &
I)
const {
129 I.setDesc(TII.get(TargetOpcode::COPY));
131 Register DstReg =
I.getOperand(0).getReg();
132 Register SrcReg =
I.getOperand(1).getReg();
134 if (isVCC(DstReg, *MRI)) {
135 if (SrcReg == AMDGPU::SCC) {
137 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
140 return RBI.constrainGenericRegister(DstReg, *RC, *MRI);
143 if (!isVCC(SrcReg, *MRI)) {
145 if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI))
149 TRI.getConstrainedRegClassForReg(SrcReg, *MRI);
151 std::optional<ValueAndVReg> ConstVal =
155 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
157 .
addImm(ConstVal->Value.getBoolValue() ? -1 : 0);
159 Register MaskedReg = MRI->createVirtualRegister(SrcRC);
166 assert(Subtarget->useRealTrue16Insts());
167 const int64_t NoMods = 0;
168 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::V_AND_B16_t16_e64), MaskedReg)
174 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::V_CMP_NE_U16_t16_e64), DstReg)
181 bool IsSGPR = TRI.isSGPRClass(SrcRC);
182 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
189 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg)
195 if (!MRI->getRegClassOrNull(SrcReg))
196 MRI->setRegClass(SrcReg, SrcRC);
202 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
203 if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI))
209 for (
const MachineOperand &MO :
I.operands()) {
210 if (MO.getReg().isPhysical())
214 TRI.getConstrainedRegClassForReg(MO.getReg(), *MRI);
217 RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI);
222bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(
MachineInstr &
I)
const {
225 Register VCCReg =
I.getOperand(1).getReg();
229 if (STI.hasScalarCompareEq64()) {
231 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
234 Register DeadDst = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
235 Cmp =
BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::S_OR_B64), DeadDst)
242 Register DstReg =
I.getOperand(0).getReg();
246 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
249bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(
MachineInstr &
I)
const {
253 Register DstReg =
I.getOperand(0).getReg();
254 Register SrcReg =
I.getOperand(1).getReg();
255 std::optional<ValueAndVReg> Arg =
259 const int64_t
Value = Arg->Value.getZExtValue();
261 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
268 return RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI);
274 unsigned SelectOpcode =
275 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
285bool AMDGPUInstructionSelector::selectReadAnyLane(
MachineInstr &
I)
const {
286 Register DstReg =
I.getOperand(0).getReg();
287 Register SrcReg =
I.getOperand(1).getReg();
292 auto RFL =
BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
300bool AMDGPUInstructionSelector::selectPHI(
MachineInstr &
I)
const {
301 const Register DefReg =
I.getOperand(0).getReg();
302 const LLT DefTy = MRI->getType(DefReg);
314 MRI->getRegClassOrRegBank(DefReg);
325 DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB);
334 for (
unsigned i = 1; i !=
I.getNumOperands(); i += 2) {
335 const Register SrcReg =
I.getOperand(i).getReg();
337 const RegisterBank *RB = MRI->getRegBankOrNull(SrcReg);
339 const LLT SrcTy = MRI->getType(SrcReg);
341 TRI.getRegClassForTypeOnBank(SrcTy, *RB);
342 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
347 I.setDesc(TII.get(TargetOpcode::PHI));
348 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
354 unsigned SubIdx)
const {
358 Register DstReg = MRI->createVirtualRegister(&SubRC);
361 unsigned ComposedSubIdx = TRI.composeSubRegIndices(MO.
getSubReg(), SubIdx);
363 BuildMI(*BB,
MI,
MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
389 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
391 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
393 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
399bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(
MachineInstr &
I)
const {
400 Register DstReg =
I.getOperand(0).getReg();
403 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
404 if (DstRB->
getID() != AMDGPU::SGPRRegBankID &&
405 DstRB->
getID() != AMDGPU::VCCRegBankID)
408 bool Is64 =
Size > 32 || (DstRB->
getID() == AMDGPU::VCCRegBankID &&
421bool AMDGPUInstructionSelector::selectG_ADD_SUB(
MachineInstr &
I)
const {
424 Register DstReg =
I.getOperand(0).getReg();
426 LLT Ty = MRI->getType(DstReg);
431 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
432 const bool IsSALU = DstRB->
getID() == AMDGPU::SGPRRegBankID;
433 const bool Sub =
I.getOpcode() == TargetOpcode::G_SUB;
437 const unsigned Opc =
Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
440 .
add(
I.getOperand(1))
441 .
add(
I.getOperand(2))
448 if (STI.hasAddNoCarryInsts()) {
449 const unsigned Opc =
Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
450 I.setDesc(TII.get(
Opc));
457 const unsigned Opc =
Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
459 Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass());
463 .
add(
I.getOperand(1))
464 .
add(
I.getOperand(2))
471 assert(!
Sub &&
"illegal sub should not reach here");
474 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
476 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
478 MachineOperand Lo1(getSubOperand64(
I.getOperand(1), HalfRC, AMDGPU::sub0));
479 MachineOperand Lo2(getSubOperand64(
I.getOperand(2), HalfRC, AMDGPU::sub0));
480 MachineOperand Hi1(getSubOperand64(
I.getOperand(1), HalfRC, AMDGPU::sub1));
481 MachineOperand Hi2(getSubOperand64(
I.getOperand(2), HalfRC, AMDGPU::sub1));
483 Register DstLo = MRI->createVirtualRegister(&HalfRC);
484 Register DstHi = MRI->createVirtualRegister(&HalfRC);
487 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::S_ADD_U32), DstLo)
490 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::S_ADDC_U32), DstHi)
496 Register CarryReg = MRI->createVirtualRegister(CarryRC);
497 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::V_ADD_CO_U32_e64), DstLo)
503 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi)
513 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
520 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI))
527bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
532 Register Dst0Reg =
I.getOperand(0).getReg();
533 Register Dst1Reg =
I.getOperand(1).getReg();
534 const bool IsAdd =
I.getOpcode() == AMDGPU::G_UADDO ||
535 I.getOpcode() == AMDGPU::G_UADDE;
536 const bool HasCarryIn =
I.getOpcode() == AMDGPU::G_UADDE ||
537 I.getOpcode() == AMDGPU::G_USUBE;
539 if (isVCC(Dst1Reg, *MRI)) {
540 unsigned NoCarryOpc =
541 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
542 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
543 I.setDesc(TII.get(HasCarryIn ? CarryOpc : NoCarryOpc));
550 Register Src0Reg =
I.getOperand(2).getReg();
551 Register Src1Reg =
I.getOperand(3).getReg();
554 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
555 .
addReg(
I.getOperand(4).getReg());
558 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
559 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
561 auto CarryInst =
BuildMI(*BB, &
I,
DL, TII.get(HasCarryIn ? CarryOpc : NoCarryOpc), Dst0Reg)
562 .
add(
I.getOperand(2))
563 .
add(
I.getOperand(3));
565 if (MRI->use_nodbg_empty(Dst1Reg)) {
566 CarryInst.setOperandDead(3);
568 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), Dst1Reg)
570 if (!MRI->getRegClassOrNull(Dst1Reg))
571 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass);
574 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) ||
575 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) ||
576 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI))
580 !RBI.constrainGenericRegister(
I.getOperand(4).getReg(),
581 AMDGPU::SReg_32RegClass, *MRI))
588bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
592 const bool IsUnsigned =
I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
593 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
594 MRI->use_nodbg_empty(
I.getOperand(1).getReg());
597 if (Subtarget->hasMADIntraFwdBug())
598 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
599 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
601 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
602 : AMDGPU::V_MAD_NC_I64_I32_e64;
604 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
609 I.setDesc(TII.get(
Opc));
611 I.addImplicitDefUseOperands(*
MF);
612 I.getOperand(0).setIsEarlyClobber(
true);
618bool AMDGPUInstructionSelector::selectG_EXTRACT(
MachineInstr &
I)
const {
620 Register DstReg =
I.getOperand(0).getReg();
621 Register SrcReg =
I.getOperand(1).getReg();
622 LLT DstTy = MRI->getType(DstReg);
623 LLT SrcTy = MRI->getType(SrcReg);
628 int64_t
Offset =
I.getOperand(2).getImm();
629 if (
Offset % 32 != 0 || DstSize > 128)
638 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
639 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
642 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI);
644 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank);
649 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
654 *SrcRC,
I.getOperand(1));
656 BuildMI(*BB, &
I,
DL, TII.get(TargetOpcode::COPY), DstReg)
657 .
addReg(SrcReg, {}, SubReg);
663bool AMDGPUInstructionSelector::selectS16MergeToS32(
MachineInstr &
MI)
const {
668 LLT Src0Ty = MRI->getType(Src0);
669 LLT Src1Ty = MRI->getType(Src1);
671 const RegisterBank *DstBank = RBI.getRegBank(Dst, *MRI, TRI);
672 const RegisterBank *Src0Bank = RBI.getRegBank(Src0, *MRI, TRI);
673 const RegisterBank *Src1Bank = RBI.getRegBank(Src1, *MRI, TRI);
674 const bool IsVector = DstBank->
getID() == AMDGPU::VGPRRegBankID;
685 if (Src0Bank->
getID() == AMDGPU::VGPRRegBankID &&
686 Src1Bank->
getID() == AMDGPU::VGPRRegBankID &&
688 BuildMI(*BB,
MI,
DL, TII.get(TargetOpcode::REG_SEQUENCE), Dst)
694 if (!RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI))
697 MI.eraseFromParent();
702 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
703 auto MIB =
BuildMI(*BB,
MI,
DL, TII.get(AMDGPU::V_AND_B32_e32), TmpReg)
708 MIB =
BuildMI(*BB,
MI,
DL, TII.get(AMDGPU::V_LSHL_OR_B32_e64), Dst)
714 MI.eraseFromParent();
737 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
738 if (Shift0 && Shift1) {
739 Opc = AMDGPU::S_PACK_HH_B32_B16;
740 MI.getOperand(1).setReg(ShiftSrc0);
741 MI.getOperand(2).setReg(ShiftSrc1);
743 Opc = AMDGPU::S_PACK_LH_B32_B16;
744 MI.getOperand(2).setReg(ShiftSrc1);
748 if (ConstSrc1 && ConstSrc1->Value == 0) {
750 auto MIB =
BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::S_LSHR_B32), Dst)
755 MI.eraseFromParent();
759 if (STI.hasSPackHL()) {
760 Opc = AMDGPU::S_PACK_HL_B32_B16;
761 MI.getOperand(1).setReg(ShiftSrc0);
765 MI.setDesc(TII.get(
Opc));
772bool AMDGPUInstructionSelector::selectS16MergeToWide(
MachineInstr &
MI)
const {
776 const uint64_t DstSize = MRI->getType(DstReg).getSizeInBits();
777 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
778 const unsigned NumSrc =
MI.getNumOperands() - 1;
782 for (
unsigned I = 0;
I != NumSrc;
I += 2) {
783 Register S32 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
793 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
794 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
796 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, 4);
797 auto MIB =
BuildMI(*BB,
MI,
DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg);
798 for (
size_t I = 0,
E = S32Regs.
size();
I !=
E; ++
I)
799 MIB.addReg(S32Regs[
I]).addImm(SubRegs[
I]);
801 MI.eraseFromParent();
805bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(
MachineInstr &
MI)
const {
808 LLT DstTy = MRI->getType(DstReg);
809 LLT SrcTy = MRI->getType(
MI.getOperand(1).getReg());
815 MI.getNumOperands() == 3) {
816 return selectS16MergeToS32(
MI);
820 bool IsWideS16Merge = SrcSize == 16 && DstTy.
getSizeInBits() > 32 &&
824 if (IsWideS16Merge &&
825 RBI.getRegBank(DstReg, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
826 return selectS16MergeToWide(
MI);
834 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
837 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
841 unsigned SrcSizeEnc =
static_cast<unsigned>(SrcSize);
842 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, SrcSizeEnc / 8);
843 MachineInstrBuilder MIB =
844 BuildMI(*BB, &
MI,
DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg);
845 for (
int I = 0,
E =
MI.getNumOperands() - 1;
I !=
E; ++
I) {
846 MachineOperand &Src =
MI.getOperand(
I + 1);
852 TRI.getConstrainedRegClassForReg(SrcReg, *MRI);
853 if (SrcRC && !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
857 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
860 MI.eraseFromParent();
864bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(
MachineInstr &
MI)
const {
866 const int NumDst =
MI.getNumOperands() - 1;
868 MachineOperand &Src =
MI.getOperand(NumDst);
872 LLT DstTy = MRI->getType(DstReg0);
873 LLT SrcTy = MRI->getType(SrcReg);
878 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI);
881 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank);
886 unsigned DstSizeEnc =
static_cast<unsigned>(DstSize);
887 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSizeEnc / 8);
888 for (
int I = 0,
E = NumDst;
I !=
E; ++
I) {
891 if (SrcBank->
getID() == AMDGPU::SGPRRegBankID &&
892 SubRegs[
I] == AMDGPU::hi16) {
893 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::S_LSHR_B32), DstReg)
897 BuildMI(*BB, &
MI,
DL, TII.get(TargetOpcode::COPY), DstReg)
898 .
addReg(SrcReg, {}, SubRegs[
I]);
901 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[
I]);
904 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
908 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
909 if (DstRC && !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
913 MI.eraseFromParent();
917bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(
MachineInstr &
MI)
const {
918 assert(
MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
919 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
923 LLT SrcTy = MRI->getType(Src0);
927 if (
MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
928 return selectG_MERGE_VALUES(
MI);
935 (
MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
939 const RegisterBank *DstBank = RBI.getRegBank(Dst, *MRI, TRI);
940 if (DstBank->
getID() == AMDGPU::AGPRRegBankID)
943 assert(DstBank->
getID() == AMDGPU::SGPRRegBankID ||
944 DstBank->
getID() == AMDGPU::VGPRRegBankID);
945 const bool IsVector = DstBank->
getID() == AMDGPU::VGPRRegBankID;
958 const int64_t K0 = ConstSrc0->Value.getSExtValue();
959 const int64_t K1 = ConstSrc1->Value.getSExtValue();
960 uint32_t
Lo16 =
static_cast<uint32_t
>(K0) & 0xffff;
961 uint32_t
Hi16 =
static_cast<uint32_t
>(K1) & 0xffff;
967 MI.eraseFromParent();
968 return RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI);
973 MI.eraseFromParent();
974 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI);
985 if (Src1Def->
getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
986 MI.setDesc(TII.get(AMDGPU::COPY));
989 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
990 return RBI.constrainGenericRegister(Dst, RC, *MRI) &&
991 RBI.constrainGenericRegister(Src0, RC, *MRI);
994 return selectS16MergeToS32(
MI);
997bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(
MachineInstr &
I)
const {
998 const MachineOperand &MO =
I.getOperand(0);
1003 TRI.getConstrainedRegClassForReg(MO.
getReg(), *MRI);
1004 if ((!RC && !MRI->getRegBankOrNull(MO.
getReg())) ||
1005 (RC && RBI.constrainGenericRegister(MO.
getReg(), *RC, *MRI))) {
1006 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
1013bool AMDGPUInstructionSelector::selectG_INSERT(
MachineInstr &
I)
const {
1016 Register DstReg =
I.getOperand(0).getReg();
1017 Register Src0Reg =
I.getOperand(1).getReg();
1018 Register Src1Reg =
I.getOperand(2).getReg();
1019 LLT Src1Ty = MRI->getType(Src1Reg);
1021 uint64_t DstSize = MRI->getType(DstReg).getSizeInBits();
1024 int64_t
Offset =
I.getOperand(3).getImm();
1027 if (
Offset % 32 != 0 || InsSize % 32 != 0)
1034 unsigned SubReg = TRI.getSubRegFromChannel(
Offset / 32, InsSize / 32);
1035 if (SubReg == AMDGPU::NoSubRegister)
1038 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
1040 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
1044 const RegisterBank *Src0Bank = RBI.getRegBank(Src0Reg, *MRI, TRI);
1045 const RegisterBank *Src1Bank = RBI.getRegBank(Src1Reg, *MRI, TRI);
1047 TRI.getRegClassForSizeOnBank(DstSize, *Src0Bank);
1049 TRI.getRegClassForSizeOnBank(InsSize, *Src1Bank);
1053 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1054 if (!Src0RC || !Src1RC)
1057 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
1058 !RBI.constrainGenericRegister(Src0Reg, *Src0RC, *MRI) ||
1059 !RBI.constrainGenericRegister(Src1Reg, *Src1RC, *MRI))
1063 BuildMI(*BB, &
I,
DL, TII.get(TargetOpcode::INSERT_SUBREG), DstReg)
1068 I.eraseFromParent();
1072bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(
MachineInstr &
MI)
const {
1075 Register OffsetReg =
MI.getOperand(2).getReg();
1076 Register WidthReg =
MI.getOperand(3).getReg();
1078 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1079 "scalar BFX instructions are expanded in regbankselect");
1080 assert(MRI->getType(
MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1081 "64-bit vector BFX instructions are expanded in regbankselect");
1086 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SBFX;
1087 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1092 MI.eraseFromParent();
1097bool AMDGPUInstructionSelector::selectInterpP1F16(
MachineInstr &
MI)
const {
1098 if (STI.getLDSBankCount() != 16)
1104 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) ||
1105 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) ||
1106 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI))
1116 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1122 BuildMI(*
MBB, &
MI,
DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov)
1125 .
addImm(
MI.getOperand(3).getImm());
1138 MI.eraseFromParent();
1147bool AMDGPUInstructionSelector::selectWritelane(
MachineInstr &
MI)
const {
1149 if (STI.getConstantBusLimit(AMDGPU::V_WRITELANE_B32) > 1)
1156 Register LaneSelect =
MI.getOperand(3).getReg();
1159 auto MIB =
BuildMI(*
MBB, &
MI,
DL, TII.get(AMDGPU::V_WRITELANE_B32), VDst);
1161 std::optional<ValueAndVReg> ConstSelect =
1167 MIB.
addImm(ConstSelect->Value.getSExtValue() &
1170 std::optional<ValueAndVReg> ConstVal =
1176 static_cast<int32_t
>(ConstVal->Value.getSExtValue()),
1177 STI.hasInv2PiInlineImm())) {
1178 MIB.
addImm(ConstVal->Value.getSExtValue());
1186 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI);
1188 BuildMI(*
MBB, *MIB,
DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
1196 MI.eraseFromParent();
1203bool AMDGPUInstructionSelector::selectDivScale(
MachineInstr &
MI)
const {
1207 LLT Ty = MRI->getType(Dst0);
1210 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1212 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1223 unsigned ChooseDenom =
static_cast<unsigned>(
MI.getOperand(5).
getImm());
1238 MI.eraseFromParent();
1243bool AMDGPUInstructionSelector::selectG_INTRINSIC(
MachineInstr &
I)
const {
1245 switch (IntrinsicID) {
1246 case Intrinsic::amdgcn_if_break: {
1251 BuildMI(*BB, &
I,
I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK))
1252 .
add(
I.getOperand(0))
1253 .
add(
I.getOperand(2))
1254 .
add(
I.getOperand(3))
1257 Register DstReg =
I.getOperand(0).getReg();
1258 Register Src0Reg =
I.getOperand(2).getReg();
1259 Register Src1Reg =
I.getOperand(3).getReg();
1261 I.eraseFromParent();
1264 MRI->setRegClass(
Reg, TRI.getWaveMaskRegClass());
1268 case Intrinsic::amdgcn_interp_p1_f16:
1269 return selectInterpP1F16(
I);
1270 case Intrinsic::amdgcn_wqm:
1271 return constrainCopyLikeIntrin(
I, AMDGPU::WQM);
1272 case Intrinsic::amdgcn_softwqm:
1273 return constrainCopyLikeIntrin(
I, AMDGPU::SOFT_WQM);
1274 case Intrinsic::amdgcn_strict_wwm:
1275 case Intrinsic::amdgcn_wwm:
1276 return constrainCopyLikeIntrin(
I, AMDGPU::STRICT_WWM);
1277 case Intrinsic::amdgcn_strict_wqm:
1278 return constrainCopyLikeIntrin(
I, AMDGPU::STRICT_WQM);
1279 case Intrinsic::amdgcn_writelane:
1280 return selectWritelane(
I);
1281 case Intrinsic::amdgcn_div_scale:
1282 return selectDivScale(
I);
1283 case Intrinsic::amdgcn_ballot:
1284 return selectBallot(
I);
1285 case Intrinsic::amdgcn_reloc_constant:
1286 return selectRelocConstant(
I);
1287 case Intrinsic::amdgcn_groupstaticsize:
1288 return selectGroupStaticSize(
I);
1289 case Intrinsic::returnaddress:
1290 return selectReturnAddress(
I);
1291 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1292 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1293 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1294 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1295 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1296 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1297 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1298 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1299 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1300 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1301 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1302 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1303 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1304 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1305 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1306 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1307 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1308 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1309 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1310 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1311 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1312 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1313 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1314 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1315 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1316 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1317 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1318 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1319 return selectSMFMACIntrin(
I);
1320 case Intrinsic::amdgcn_permlane16_swap:
1321 case Intrinsic::amdgcn_permlane32_swap:
1322 return selectPermlaneSwapIntrin(
I, IntrinsicID);
1323 case Intrinsic::amdgcn_wave_shuffle:
1324 return selectWaveShuffleIntrin(
I);
1335 if (
Size == 16 && !ST.has16BitInsts())
1338 const auto Select = [&](
unsigned S16Opc,
unsigned TrueS16Opc,
1339 unsigned FakeS16Opc,
unsigned S32Opc,
1342 return ST.hasTrue16BitInsts()
1343 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1354 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1355 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1356 AMDGPU::V_CMP_NE_U64_e64);
1358 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1359 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1360 AMDGPU::V_CMP_EQ_U64_e64);
1362 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1363 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1364 AMDGPU::V_CMP_GT_I64_e64);
1366 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1367 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1368 AMDGPU::V_CMP_GE_I64_e64);
1370 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1371 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1372 AMDGPU::V_CMP_LT_I64_e64);
1374 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1375 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1376 AMDGPU::V_CMP_LE_I64_e64);
1378 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1379 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1380 AMDGPU::V_CMP_GT_U64_e64);
1382 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1383 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1384 AMDGPU::V_CMP_GE_U64_e64);
1386 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1387 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1388 AMDGPU::V_CMP_LT_U64_e64);
1390 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1391 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1392 AMDGPU::V_CMP_LE_U64_e64);
1395 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1396 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1397 AMDGPU::V_CMP_EQ_F64_e64);
1399 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1400 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1401 AMDGPU::V_CMP_GT_F64_e64);
1403 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1404 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1405 AMDGPU::V_CMP_GE_F64_e64);
1407 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1408 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1409 AMDGPU::V_CMP_LT_F64_e64);
1411 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1412 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1413 AMDGPU::V_CMP_LE_F64_e64);
1415 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1416 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1417 AMDGPU::V_CMP_NEQ_F64_e64);
1419 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1420 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1421 AMDGPU::V_CMP_O_F64_e64);
1423 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1424 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1425 AMDGPU::V_CMP_U_F64_e64);
1427 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1428 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1429 AMDGPU::V_CMP_NLG_F64_e64);
1431 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1432 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1433 AMDGPU::V_CMP_NLE_F64_e64);
1435 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1436 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1437 AMDGPU::V_CMP_NLT_F64_e64);
1439 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1440 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1441 AMDGPU::V_CMP_NGE_F64_e64);
1443 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1444 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1445 AMDGPU::V_CMP_NGT_F64_e64);
1447 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1448 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1449 AMDGPU::V_CMP_NEQ_F64_e64);
1451 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1452 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1453 AMDGPU::V_CMP_TRU_F64_e64);
1455 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1456 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1457 AMDGPU::V_CMP_F_F64_e64);
1464 if (!STI.hasScalarCompareEq64())
1469 return AMDGPU::S_CMP_LG_U64;
1471 return AMDGPU::S_CMP_EQ_U64;
1480 return AMDGPU::S_CMP_LG_U32;
1482 return AMDGPU::S_CMP_EQ_U32;
1484 return AMDGPU::S_CMP_GT_I32;
1486 return AMDGPU::S_CMP_GE_I32;
1488 return AMDGPU::S_CMP_LT_I32;
1490 return AMDGPU::S_CMP_LE_I32;
1492 return AMDGPU::S_CMP_GT_U32;
1494 return AMDGPU::S_CMP_GE_U32;
1496 return AMDGPU::S_CMP_LT_U32;
1498 return AMDGPU::S_CMP_LE_U32;
1500 return AMDGPU::S_CMP_EQ_F32;
1502 return AMDGPU::S_CMP_GT_F32;
1504 return AMDGPU::S_CMP_GE_F32;
1506 return AMDGPU::S_CMP_LT_F32;
1508 return AMDGPU::S_CMP_LE_F32;
1510 return AMDGPU::S_CMP_LG_F32;
1512 return AMDGPU::S_CMP_O_F32;
1514 return AMDGPU::S_CMP_U_F32;
1516 return AMDGPU::S_CMP_NLG_F32;
1518 return AMDGPU::S_CMP_NLE_F32;
1520 return AMDGPU::S_CMP_NLT_F32;
1522 return AMDGPU::S_CMP_NGE_F32;
1524 return AMDGPU::S_CMP_NGT_F32;
1526 return AMDGPU::S_CMP_NEQ_F32;
1533 if (!STI.hasSALUFloatInsts())
1538 return AMDGPU::S_CMP_EQ_F16;
1540 return AMDGPU::S_CMP_GT_F16;
1542 return AMDGPU::S_CMP_GE_F16;
1544 return AMDGPU::S_CMP_LT_F16;
1546 return AMDGPU::S_CMP_LE_F16;
1548 return AMDGPU::S_CMP_LG_F16;
1550 return AMDGPU::S_CMP_O_F16;
1552 return AMDGPU::S_CMP_U_F16;
1554 return AMDGPU::S_CMP_NLG_F16;
1556 return AMDGPU::S_CMP_NLE_F16;
1558 return AMDGPU::S_CMP_NLT_F16;
1560 return AMDGPU::S_CMP_NGE_F16;
1562 return AMDGPU::S_CMP_NGT_F16;
1564 return AMDGPU::S_CMP_NEQ_F16;
1573bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(
MachineInstr &
I)
const {
1578 Register SrcReg =
I.getOperand(2).getReg();
1583 Register CCReg =
I.getOperand(0).getReg();
1584 if (!isVCC(CCReg, *MRI)) {
1585 int Opcode = getS_CMPOpcode(
Pred,
Size);
1588 MachineInstr *ICmp =
BuildMI(*BB, &
I,
DL, TII.get(Opcode))
1589 .
add(
I.getOperand(2))
1590 .
add(
I.getOperand(3));
1591 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), CCReg)
1595 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI);
1596 I.eraseFromParent();
1600 if (
I.getOpcode() == AMDGPU::G_FCMP)
1607 MachineInstrBuilder ICmp;
1610 ICmp =
BuildMI(*BB, &
I,
DL, TII.get(Opcode),
I.getOperand(0).getReg())
1612 .
add(
I.getOperand(2))
1614 .
add(
I.getOperand(3))
1617 ICmp =
BuildMI(*BB, &
I,
DL, TII.get(Opcode),
I.getOperand(0).getReg())
1618 .
add(
I.getOperand(2))
1619 .
add(
I.getOperand(3));
1623 *TRI.getBoolRC(), *MRI);
1625 I.eraseFromParent();
1636 if (
MI->getParent() !=
MBB)
1640 if (
MI->getOpcode() == AMDGPU::COPY) {
1641 auto DstRB =
MRI.getRegBankOrNull(
MI->getOperand(0).getReg());
1642 auto SrcRB =
MRI.getRegBankOrNull(
MI->getOperand(1).getReg());
1643 if (DstRB && SrcRB && DstRB->
getID() == AMDGPU::VCCRegBankID &&
1644 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1649 if (
MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1665bool AMDGPUInstructionSelector::selectBallot(
MachineInstr &
I)
const {
1668 Register DstReg =
I.getOperand(0).getReg();
1669 Register SrcReg =
I.getOperand(2).getReg();
1670 const uint64_t BallotSize = MRI->getType(DstReg).getSizeInBits();
1671 const unsigned WaveSize = STI.getWavefrontSize();
1675 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1678 std::optional<ValueAndVReg> Arg =
1683 if (BallotSize != WaveSize) {
1684 Dst = MRI->createVirtualRegister(TRI.getBoolRC());
1688 const int64_t
Value = Arg->Value.getZExtValue();
1691 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1698 if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI))
1704 if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI))
1708 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1718 if (BallotSize != WaveSize) {
1719 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1721 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
1728 I.eraseFromParent();
1732bool AMDGPUInstructionSelector::selectRelocConstant(
MachineInstr &
I)
const {
1733 Register DstReg =
I.getOperand(0).getReg();
1734 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
1736 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
1739 const bool IsVALU = DstBank->
getID() == AMDGPU::VGPRRegBankID;
1741 Module *
M =
MF->getFunction().getParent();
1742 const MDNode *
Metadata =
I.getOperand(2).getMetadata();
1749 TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg)
1752 I.eraseFromParent();
1756bool AMDGPUInstructionSelector::selectGroupStaticSize(
MachineInstr &
I)
const {
1759 Register DstReg =
I.getOperand(0).getReg();
1760 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
1761 unsigned Mov = DstRB->
getID() == AMDGPU::SGPRRegBankID ?
1762 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1770 const SIMachineFunctionInfo *MFI =
MF->getInfo<SIMachineFunctionInfo>();
1773 Module *
M =
MF->getFunction().getParent();
1774 const GlobalValue *GV =
1779 I.eraseFromParent();
1784bool AMDGPUInstructionSelector::selectReturnAddress(
MachineInstr &
I)
const {
1789 Register DstReg =
I.getOperand(0).getReg();
1790 unsigned Depth =
static_cast<unsigned>(
I.getOperand(2).
getImm());
1793 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
1795 !RBI.constrainGenericRegister(DstReg, *RC, *MRI))
1800 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1803 I.eraseFromParent();
1807 MachineFrameInfo &MFI =
MF.getFrameInfo();
1812 Register ReturnAddrReg = TRI.getReturnAddressReg(
MF);
1814 AMDGPU::SReg_64RegClass,
DL);
1817 I.eraseFromParent();
1821bool AMDGPUInstructionSelector::selectEndCfIntrinsic(
MachineInstr &
MI)
const {
1825 BuildMI(*BB, &
MI,
MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF))
1826 .
add(
MI.getOperand(1))
1830 MI.eraseFromParent();
1832 if (!MRI->getRegClassOrNull(
Reg))
1833 MRI->setRegClass(
Reg, TRI.getWaveMaskRegClass());
1837bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1843 unsigned IndexOperand =
static_cast<unsigned>(
MI.getOperand(7).
getImm());
1844 bool WaveRelease =
MI.getOperand(8).getImm() != 0;
1845 bool WaveDone =
MI.getOperand(9).getImm() != 0;
1847 if (WaveDone && !WaveRelease) {
1851 Fn,
"ds_ordered_count: wave_done requires wave_release",
DL));
1854 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1855 IndexOperand &= ~0x3f;
1856 unsigned CountDw = 0;
1859 CountDw = (IndexOperand >> 24) & 0xf;
1860 IndexOperand &= ~(0xf << 24);
1862 if (CountDw < 1 || CountDw > 4) {
1865 Fn,
"ds_ordered_count: dword count must be between 1 and 4",
DL));
1873 Fn,
"ds_ordered_count: bad index operand",
DL));
1876 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1879 unsigned Offset0 = OrderedCountIndex << 2;
1880 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1883 Offset1 |= (CountDw - 1) << 6;
1886 Offset1 |= ShaderType << 2;
1888 unsigned Offset = Offset0 | (Offset1 << 8);
1896 MachineInstrBuilder
DS =
1897 BuildMI(*
MBB, &
MI,
DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg)
1902 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI))
1906 MI.eraseFromParent();
1912 case Intrinsic::amdgcn_ds_gws_init:
1913 return AMDGPU::DS_GWS_INIT;
1914 case Intrinsic::amdgcn_ds_gws_barrier:
1915 return AMDGPU::DS_GWS_BARRIER;
1916 case Intrinsic::amdgcn_ds_gws_sema_v:
1917 return AMDGPU::DS_GWS_SEMA_V;
1918 case Intrinsic::amdgcn_ds_gws_sema_br:
1919 return AMDGPU::DS_GWS_SEMA_BR;
1920 case Intrinsic::amdgcn_ds_gws_sema_p:
1921 return AMDGPU::DS_GWS_SEMA_P;
1922 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1923 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1929bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(
MachineInstr &
MI,
1931 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1932 !STI.hasGWSSemaReleaseAll()))
1936 const bool HasVSrc =
MI.getNumOperands() == 3;
1937 assert(HasVSrc ||
MI.getNumOperands() == 2);
1939 Register BaseOffset =
MI.getOperand(HasVSrc ? 2 : 1).getReg();
1940 const RegisterBank *OffsetRB = RBI.getRegBank(BaseOffset, *MRI, TRI);
1941 if (OffsetRB->
getID() != AMDGPU::SGPRRegBankID)
1950 MachineInstr *Readfirstlane =
nullptr;
1955 if (OffsetDef->
getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1956 Readfirstlane = OffsetDef;
1961 if (OffsetDef->
getOpcode() == AMDGPU::G_CONSTANT) {
1967 ImmOffset =
static_cast<unsigned>(
1968 OffsetDef->
getOperand(1).getCImm()->getZExtValue());
1972 std::tie(BaseOffset, ImmOffset) =
1975 if (Readfirstlane) {
1978 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI))
1984 if (!RBI.constrainGenericRegister(BaseOffset,
1985 AMDGPU::SReg_32RegClass, *MRI))
1989 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2004 const MCInstrDesc &InstrDesc = TII.get(
Opc);
2009 int Data0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::data0);
2012 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2016 if (!RBI.constrainGenericRegister(VSrc, *DataRC, *MRI))
2026 Register DataReg = MRI->createVirtualRegister(DataRC);
2027 if (!RBI.constrainGenericRegister(VSrc, *SubRC, *MRI))
2030 Register UndefReg = MRI->createVirtualRegister(SubRC);
2049 MI.eraseFromParent();
2053bool AMDGPUInstructionSelector::selectDSAppendConsume(
MachineInstr &
MI,
2054 bool IsAppend)
const {
2055 Register PtrBase =
MI.getOperand(2).getReg();
2056 LLT PtrTy = MRI->getType(PtrBase);
2060 std::tie(PtrBase,
Offset) = selectDS1Addr1OffsetImpl(
MI.getOperand(2));
2063 if (!isDSOffsetLegal(PtrBase,
Offset)) {
2064 PtrBase =
MI.getOperand(2).getReg();
2070 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2074 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI))
2081 MI.eraseFromParent();
2086bool AMDGPUInstructionSelector::selectInitWholeWave(
MachineInstr &
MI)
const {
2088 SIMachineFunctionInfo *MFInfo =
MF->getInfo<SIMachineFunctionInfo>();
2099 TFE = TexFailCtrl & 0x1;
2101 LWE = TexFailCtrl & 0x2;
2104 return TexFailCtrl == 0;
2107bool AMDGPUInstructionSelector::selectImageIntrinsic(
2115 Register ResultDef =
MI.getOperand(0).getReg();
2116 if (MRI->use_nodbg_empty(ResultDef))
2120 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2129 const unsigned ArgOffset =
MI.getNumExplicitDefs() + 1;
2131 Register VDataIn = AMDGPU::NoRegister;
2132 Register VDataOut = AMDGPU::NoRegister;
2134 int NumVDataDwords = -1;
2135 bool IsD16 =
MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2136 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2142 Unorm =
MI.getOperand(ArgOffset + Intr->
UnormIndex).getImm() != 0;
2146 bool IsTexFail =
false;
2148 TFE, LWE, IsTexFail))
2151 const int64_t
Flags =
MI.getOperand(ArgOffset + Intr->
NumArgs).getImm();
2152 const bool IsA16 = (
Flags & 1) != 0;
2153 const bool IsG16 = (
Flags & 2) != 0;
2156 if (IsA16 && !STI.hasG16() && !IsG16)
2160 unsigned DMaskLanes = 0;
2162 if (BaseOpcode->
Atomic) {
2164 VDataOut =
MI.getOperand(0).getReg();
2165 VDataIn =
MI.getOperand(2).getReg();
2166 LLT Ty = MRI->getType(VDataIn);
2169 const bool Is64Bit = BaseOpcode->
AtomicX2 ?
2174 assert(
MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2176 DMask = Is64Bit ? 0xf : 0x3;
2177 NumVDataDwords = Is64Bit ? 4 : 2;
2179 DMask = Is64Bit ? 0x3 : 0x1;
2180 NumVDataDwords = Is64Bit ? 2 : 1;
2183 DMask =
static_cast<unsigned>(
2187 if (BaseOpcode->
Store) {
2188 VDataIn =
MI.getOperand(1).getReg();
2189 VDataTy = MRI->getType(VDataIn);
2190 NumVDataDwords =
static_cast<int>((VDataTy.
getSizeInBits() + 31) / 32);
2194 VDataOut =
MI.getOperand(0).getReg();
2195 VDataTy = MRI->getType(VDataOut);
2196 NumVDataDwords = DMaskLanes;
2198 if (IsD16 && !STI.hasUnpackedD16VMem())
2199 NumVDataDwords = (DMaskLanes + 1) / 2;
2204 if (Subtarget->hasG16() && IsG16) {
2205 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2208 IntrOpcode = G16MappingInfo->
G16;
2212 assert((!IsTexFail || DMaskLanes >= 1) &&
"should have legalized this");
2214 unsigned CPol =
static_cast<unsigned>(
2223 int NumVAddrRegs = 0;
2224 int NumVAddrDwords = 0;
2227 MachineOperand &AddrOp =
MI.getOperand(ArgOffset +
I);
2228 if (!AddrOp.
isReg())
2237 static_cast<int>((MRI->getType(Addr).getSizeInBits() + 31) / 32);
2244 NumVAddrRegs != 1 &&
2245 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2246 : NumVAddrDwords == NumVAddrRegs);
2247 if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) {
2259 bool IndexedRsrc = MRI->getType(RsrcReg) ==
I32;
2260 bool IndexedSamp =
false;
2263 IndexedSamp = MRI->getType(SampReg) ==
I32;
2267 NumVAddrDwords, IndexedRsrc, IndexedSamp);
2268 }
else if (IsGFX12Plus) {
2270 NumVDataDwords, NumVAddrDwords);
2271 }
else if (IsGFX11Plus) {
2273 UseNSA ? AMDGPU::MIMGEncGfx11NSA
2274 : AMDGPU::MIMGEncGfx11Default,
2275 NumVDataDwords, NumVAddrDwords);
2276 }
else if (IsGFX10Plus) {
2278 UseNSA ? AMDGPU::MIMGEncGfx10NSA
2279 : AMDGPU::MIMGEncGfx10Default,
2280 NumVDataDwords, NumVAddrDwords);
2282 if (Subtarget->hasGFX90AInsts()) {
2284 NumVDataDwords, NumVAddrDwords);
2288 <<
"requested image instruction is not supported on this GPU\n");
2295 NumVDataDwords, NumVAddrDwords);
2298 NumVDataDwords, NumVAddrDwords);
2308 const bool Is64 = MRI->getType(VDataOut).getSizeInBits() == 64;
2310 Register TmpReg = MRI->createVirtualRegister(
2311 Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2312 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2315 if (!MRI->use_empty(VDataOut)) {
2317 .
addReg(TmpReg, {}, SubReg);
2328 for (
int I = 0;
I != NumVAddrRegs; ++
I) {
2329 MachineOperand &SrcOp =
MI.getOperand(ArgOffset + Intr->
VAddrStart +
I);
2330 if (SrcOp.
isReg()) {
2349 STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0);
2351 MIB.
addImm(IsA16 ? -1 : 0);
2353 if (!Subtarget->hasGFX90AInsts()) {
2365 MIB.
addImm(IsD16 ? -1 : 0);
2367 MI.eraseFromParent();
2369 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr);
2375bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2386 unsigned Offset =
static_cast<unsigned>(
MI.getOperand(6).
getImm());
2390 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2391 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2392 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2394 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2395 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2397 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2398 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2410 MI.eraseFromParent();
2415bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2418 if (
const AMDGPU::BufferIntrinsicInfo *BufInfo =
2420 BufInfo && BufInfo->
Kind == AMDGPU::BufferKindLoadToLDS)
2421 return selectBufferLoadLds(
I, *BufInfo);
2423 switch (IntrinsicID) {
2424 case Intrinsic::amdgcn_end_cf:
2425 return selectEndCfIntrinsic(
I);
2426 case Intrinsic::amdgcn_ds_ordered_add:
2427 case Intrinsic::amdgcn_ds_ordered_swap:
2428 return selectDSOrderedIntrinsic(
I, IntrinsicID);
2429 case Intrinsic::amdgcn_ds_gws_init:
2430 case Intrinsic::amdgcn_ds_gws_barrier:
2431 case Intrinsic::amdgcn_ds_gws_sema_v:
2432 case Intrinsic::amdgcn_ds_gws_sema_br:
2433 case Intrinsic::amdgcn_ds_gws_sema_p:
2434 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2435 return selectDSGWSIntrinsic(
I, IntrinsicID);
2436 case Intrinsic::amdgcn_ds_append:
2437 return selectDSAppendConsume(
I,
true);
2438 case Intrinsic::amdgcn_ds_consume:
2439 return selectDSAppendConsume(
I,
false);
2440 case Intrinsic::amdgcn_init_whole_wave:
2441 return selectInitWholeWave(
I);
2446 case Intrinsic::amdgcn_load_to_lds:
2447 case Intrinsic::amdgcn_load_async_to_lds:
2448 case Intrinsic::amdgcn_global_load_lds:
2449 case Intrinsic::amdgcn_global_load_async_lds:
2450 return selectGlobalLoadLds(
I);
2451 case Intrinsic::amdgcn_tensor_load_to_lds:
2452 case Intrinsic::amdgcn_tensor_store_from_lds:
2453 return selectTensorLoadStore(
I, IntrinsicID);
2454 case Intrinsic::amdgcn_asyncmark:
2455 case Intrinsic::amdgcn_wait_asyncmark:
2456 if (!Subtarget->hasAsyncMark())
2459 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2460 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2461 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2462 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2463 return selectDSBvhStackIntrinsic(
I);
2464 case Intrinsic::amdgcn_s_alloc_vgpr: {
2470 Register ResReg =
I.getOperand(0).getReg();
2472 MachineInstr *AllocMI =
BuildMI(*
MBB, &
I,
DL, TII.get(AMDGPU::S_ALLOC_VGPR))
2473 .
add(
I.getOperand(2));
2476 I.eraseFromParent();
2478 return RBI.constrainGenericRegister(ResReg, AMDGPU::SReg_32RegClass, *MRI);
2480 case Intrinsic::amdgcn_s_barrier_init:
2481 case Intrinsic::amdgcn_s_barrier_signal_var:
2482 return selectNamedBarrierInit(
I, IntrinsicID);
2483 case Intrinsic::amdgcn_s_wakeup_barrier:
2484 case Intrinsic::amdgcn_s_barrier_join:
2485 case Intrinsic::amdgcn_s_get_named_barrier_state:
2486 return selectNamedBarrierInst(
I, IntrinsicID);
2487 case Intrinsic::amdgcn_s_get_barrier_state:
2488 return selectSGetBarrierState(
I, IntrinsicID);
2489 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2490 return selectSBarrierSignalIsfirst(
I, IntrinsicID);
2495bool AMDGPUInstructionSelector::selectG_SELECT(
MachineInstr &
I)
const {
2502 Register DstReg =
I.getOperand(0).getReg();
2505 const MachineOperand &CCOp =
I.getOperand(1);
2507 if (!isVCC(CCReg, *MRI)) {
2508 unsigned SelectOpcode =
Size == 64 ? AMDGPU::S_CSELECT_B64 :
2509 AMDGPU::S_CSELECT_B32;
2510 MachineInstr *CopySCC =
BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
2516 if (!MRI->getRegClassOrNull(CCReg))
2517 MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForReg(CCReg, *MRI));
2519 .
add(
I.getOperand(2))
2520 .
add(
I.getOperand(3));
2524 I.eraseFromParent();
2533 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
2535 .
add(
I.getOperand(3))
2537 .
add(
I.getOperand(2))
2538 .
add(
I.getOperand(1));
2541 I.eraseFromParent();
2545bool AMDGPUInstructionSelector::selectG_TRUNC(
MachineInstr &
I)
const {
2546 Register DstReg =
I.getOperand(0).getReg();
2547 Register SrcReg =
I.getOperand(1).getReg();
2548 const LLT DstTy = MRI->getType(DstReg);
2549 const LLT SrcTy = MRI->getType(SrcReg);
2552 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
2553 const RegisterBank *DstRB;
2559 DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
2564 const bool IsVALU = DstRB->
getID() == AMDGPU::VGPRRegBankID;
2570 TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB);
2572 TRI.getRegClassForSizeOnBank(DstSize, *DstRB);
2573 if (!SrcRC || !DstRC)
2576 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) ||
2577 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
2582 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2583 assert(STI.useRealTrue16Insts());
2587 .
addReg(SrcReg, {}, AMDGPU::lo16);
2588 I.eraseFromParent();
2596 Register LoReg = MRI->createVirtualRegister(DstRC);
2597 Register HiReg = MRI->createVirtualRegister(DstRC);
2599 .
addReg(SrcReg, {}, AMDGPU::sub0);
2601 .
addReg(SrcReg, {}, AMDGPU::sub1);
2603 if (IsVALU && STI.hasSDWA()) {
2606 MachineInstr *MovSDWA =
2607 BuildMI(*
MBB,
I,
DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg)
2617 Register TmpReg0 = MRI->createVirtualRegister(DstRC);
2618 Register TmpReg1 = MRI->createVirtualRegister(DstRC);
2619 Register ImmReg = MRI->createVirtualRegister(DstRC);
2621 BuildMI(*
MBB,
I,
DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), TmpReg0)
2631 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2632 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2633 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2645 And.setOperandDead(3);
2646 Or.setOperandDead(3);
2650 I.eraseFromParent();
2658 unsigned SubRegIdx = DstSize < 32
2659 ?
static_cast<unsigned>(AMDGPU::sub0)
2660 : TRI.getSubRegFromChannel(0, DstSize / 32);
2661 if (SubRegIdx == AMDGPU::NoSubRegister)
2667 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2671 if (SrcWithSubRC != SrcRC) {
2672 if (!RBI.constrainGenericRegister(SrcReg, *SrcWithSubRC, *MRI))
2676 I.getOperand(1).setSubReg(SubRegIdx);
2679 I.setDesc(TII.get(TargetOpcode::COPY));
2686 int SignedMask =
static_cast<int>(Mask);
2687 return SignedMask >= -16 && SignedMask <= 64;
2691const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2700 return &RBI.getRegBankFromRegClass(*RC, LLT());
2704bool AMDGPUInstructionSelector::selectG_SZA_EXT(
MachineInstr &
I)
const {
2705 bool InReg =
I.getOpcode() == AMDGPU::G_SEXT_INREG;
2706 bool Signed =
I.getOpcode() == AMDGPU::G_SEXT || InReg;
2709 const Register DstReg =
I.getOperand(0).getReg();
2710 const Register SrcReg =
I.getOperand(1).getReg();
2712 const LLT DstTy = MRI->getType(DstReg);
2713 const LLT SrcTy = MRI->getType(SrcReg);
2714 const uint64_t SrcSize =
I.getOpcode() == AMDGPU::G_SEXT_INREG
2715 ?
I.getOperand(2).getImm()
2722 const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI);
2725 if (
I.getOpcode() == AMDGPU::G_ANYEXT) {
2727 return selectCOPY(
I);
2730 TRI.getRegClassForTypeOnBank(SrcTy, *SrcBank);
2731 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
2733 TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
2735 Register UndefReg = MRI->createVirtualRegister(SrcRC);
2736 BuildMI(
MBB,
I,
DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
2742 I.eraseFromParent();
2744 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) &&
2745 RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI);
2748 if (SrcBank->
getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2754 MachineInstr *ExtI =
2758 I.eraseFromParent();
2763 const unsigned BFE =
Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2764 MachineInstr *ExtI =
2769 I.eraseFromParent();
2774 if (SrcBank->
getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2776 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2777 if (!RBI.constrainGenericRegister(SrcReg, SrcRC, *MRI))
2780 if (
Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2781 const unsigned SextOpc = SrcSize == 8 ?
2782 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2785 I.eraseFromParent();
2786 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
2791 if (DstSize > 32 && SrcSize == 32) {
2792 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2793 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2796 .
addReg(SrcReg, {}, SubReg)
2804 .
addReg(SrcReg, {}, SubReg)
2805 .addImm(AMDGPU::sub0)
2808 I.eraseFromParent();
2809 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass,
2813 const unsigned BFE64 =
Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2814 const unsigned BFE32 =
Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2817 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2819 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
2820 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2821 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2823 BuildMI(
MBB,
I,
DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
2825 .
addReg(SrcReg, {}, SubReg)
2826 .addImm(AMDGPU::sub0)
2834 I.eraseFromParent();
2835 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI);
2850 I.eraseFromParent();
2851 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
2877 MRI.getType(In).getSizeInBits() == 16) {
2897 if (
MRI.getType(Result).getSizeInBits() != 32)
2910 assert(Mask.size() == 2);
2912 if (Mask[0] == 1 && Mask[1] <= 1) {
2927 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(0).getReg() == In &&
2928 MRI.getType(In).getSizeInBits() == 16) {
2937 MRI.getType(
Trunc).getSizeInBits() == 32) {
2945bool AMDGPUInstructionSelector::selectG_FPEXT(
MachineInstr &
I)
const {
2946 if (!Subtarget->hasSALUFloatInsts())
2949 Register Dst =
I.getOperand(0).getReg();
2950 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI);
2951 if (DstRB->
getID() != AMDGPU::SGPRRegBankID)
2954 Register Src =
I.getOperand(1).getReg();
2960 BuildMI(*BB, &
I,
I.getDebugLoc(), TII.get(AMDGPU::S_CVT_HI_F32_F16), Dst)
2962 I.eraseFromParent();
2963 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI);
2970bool AMDGPUInstructionSelector::selectG_FNEG(
MachineInstr &
MI)
const {
2983 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI);
2984 if (DstRB->
getID() != AMDGPU::SGPRRegBankID ||
2989 MachineInstr *Fabs =
getOpcodeDef(TargetOpcode::G_FABS, Src, *MRI);
2993 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) ||
2994 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI))
2999 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3000 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3001 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3002 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3004 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::COPY), LoReg)
3005 .
addReg(Src, {}, AMDGPU::sub0);
3006 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::COPY), HiReg)
3007 .
addReg(Src, {}, AMDGPU::sub1);
3008 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::S_MOV_B32), ConstReg)
3012 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
3017 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::REG_SEQUENCE), Dst)
3022 MI.eraseFromParent();
3027bool AMDGPUInstructionSelector::selectG_FABS(
MachineInstr &
MI)
const {
3029 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI);
3030 if (DstRB->
getID() != AMDGPU::SGPRRegBankID ||
3037 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3038 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3039 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3040 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
3042 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) ||
3043 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI))
3046 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::COPY), LoReg)
3047 .
addReg(Src, {}, AMDGPU::sub0);
3048 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::COPY), HiReg)
3049 .
addReg(Src, {}, AMDGPU::sub1);
3050 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::S_MOV_B32), ConstReg)
3055 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::S_AND_B32), OpReg)
3059 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::REG_SEQUENCE), Dst)
3065 MI.eraseFromParent();
3070 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3076 unsigned OpNo =
Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3077 const MachineInstr *PtrMI = MRI.getVRegDef(
Load.getOperand(OpNo).getReg());
3081 if (PtrMI->
getOpcode() != TargetOpcode::G_PTR_ADD)
3086 for (
unsigned i = 1; i != 3; ++i) {
3087 const MachineOperand &GEPOp = PtrMI->
getOperand(i);
3088 const MachineInstr *OpDef = MRI.getVRegDef(GEPOp.
getReg());
3093 assert(GEPInfo.Imm == 0);
3097 const RegisterBank *OpBank = RBI.getRegBank(GEPOp.
getReg(), MRI, TRI);
3098 if (OpBank->
getID() == AMDGPU::SGPRRegBankID)
3099 GEPInfo.SgprParts.push_back(GEPOp.
getReg());
3101 GEPInfo.VgprParts.push_back(GEPOp.
getReg());
3105 getAddrModeInfo(*PtrMI, MRI, AddrInfo);
3108bool AMDGPUInstructionSelector::isSGPR(
Register Reg)
const {
3109 return RBI.getRegBank(
Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3112bool AMDGPUInstructionSelector::isInstrUniform(
const MachineInstr &
MI)
const {
3113 if (!
MI.hasOneMemOperand())
3116 const MachineMemOperand *MMO = *
MI.memoperands_begin();
3129 if (
MI.getOpcode() == AMDGPU::G_PREFETCH)
3130 return RBI.getRegBank(
MI.getOperand(0).getReg(), *MRI, TRI)->getID() ==
3131 AMDGPU::SGPRRegBankID;
3134 return I &&
I->getMetadata(
"amdgpu.uniform");
3138 for (
const GEPInfo &GEPInfo : AddrInfo) {
3139 if (!GEPInfo.VgprParts.empty())
3145void AMDGPUInstructionSelector::initM0(
MachineInstr &
I)
const {
3146 const LLT PtrTy = MRI->getType(
I.getOperand(1).getReg());
3149 STI.ldsRequiresM0Init()) {
3153 BuildMI(*BB, &
I,
I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3158bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3165 if (
Reg.isPhysical())
3169 const unsigned Opcode =
MI.getOpcode();
3171 if (Opcode == AMDGPU::COPY)
3174 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3175 Opcode == AMDGPU::G_XOR)
3180 return GI->is(Intrinsic::amdgcn_class);
3182 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3185bool AMDGPUInstructionSelector::selectG_BRCOND(
MachineInstr &
I)
const {
3187 MachineOperand &CondOp =
I.getOperand(0);
3200 if (!isVCC(CondReg, *MRI)) {
3204 CondPhysReg = AMDGPU::SCC;
3205 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3206 ConstrainRC = &AMDGPU::SReg_32RegClass;
3213 const bool Is64 = STI.isWave64();
3214 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3215 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3217 Register TmpReg = MRI->createVirtualRegister(TRI.getBoolRC());
3218 BuildMI(*BB, &
I,
DL, TII.get(Opcode), TmpReg)
3225 CondPhysReg = TRI.getVCC();
3226 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3227 ConstrainRC = TRI.getBoolRC();
3230 if (!MRI->getRegClassOrNull(CondReg))
3231 MRI->setRegClass(CondReg, ConstrainRC);
3233 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), CondPhysReg)
3236 .
addMBB(
I.getOperand(1).getMBB());
3238 I.eraseFromParent();
3242bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3244 Register DstReg =
I.getOperand(0).getReg();
3245 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
3246 const bool IsVGPR = DstRB->
getID() == AMDGPU::VGPRRegBankID;
3247 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3251 return RBI.constrainGenericRegister(
3252 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI);
3255bool AMDGPUInstructionSelector::selectG_PTRMASK(
MachineInstr &
I)
const {
3256 Register DstReg =
I.getOperand(0).getReg();
3257 Register SrcReg =
I.getOperand(1).getReg();
3258 Register MaskReg =
I.getOperand(2).getReg();
3259 LLT Ty = MRI->getType(DstReg);
3260 LLT MaskTy = MRI->getType(MaskReg);
3264 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
3265 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
3266 const RegisterBank *MaskRB = RBI.getRegBank(MaskReg, *MRI, TRI);
3267 const bool IsVGPR = DstRB->
getID() == AMDGPU::VGPRRegBankID;
3273 APInt MaskOnes =
VT->getKnownOnes(MaskReg).zext(64);
3277 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3278 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3281 !CanCopyLow32 && !CanCopyHi32) {
3282 auto MIB =
BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::S_AND_B64), DstReg)
3286 I.eraseFromParent();
3291 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3293 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3298 TRI.getRegClassForTypeOnBank(MaskTy, *MaskRB);
3300 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
3301 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) ||
3302 !RBI.constrainGenericRegister(MaskReg, *MaskRC, *MRI))
3307 "ptrmask should have been narrowed during legalize");
3309 auto NewOp =
BuildMI(*BB, &
I,
DL, TII.get(NewOpc), DstReg)
3315 I.eraseFromParent();
3319 Register HiReg = MRI->createVirtualRegister(&RegRC);
3320 Register LoReg = MRI->createVirtualRegister(&RegRC);
3323 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), LoReg)
3324 .
addReg(SrcReg, {}, AMDGPU::sub0);
3325 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), HiReg)
3326 .
addReg(SrcReg, {}, AMDGPU::sub1);
3335 Register MaskLo = MRI->createVirtualRegister(&RegRC);
3336 MaskedLo = MRI->createVirtualRegister(&RegRC);
3338 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), MaskLo)
3339 .
addReg(MaskReg, {}, AMDGPU::sub0);
3340 BuildMI(*BB, &
I,
DL, TII.get(NewOpc), MaskedLo)
3349 Register MaskHi = MRI->createVirtualRegister(&RegRC);
3350 MaskedHi = MRI->createVirtualRegister(&RegRC);
3352 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::COPY), MaskHi)
3353 .
addReg(MaskReg, {}, AMDGPU::sub1);
3354 BuildMI(*BB, &
I,
DL, TII.get(NewOpc), MaskedHi)
3359 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
3364 I.eraseFromParent();
3370static std::pair<Register, unsigned>
3377 std::tie(IdxBaseReg,
Offset) =
3379 if (IdxBaseReg == AMDGPU::NoRegister) {
3383 IdxBaseReg = IdxReg;
3390 if (
static_cast<unsigned>(
Offset) >= SubRegs.
size())
3391 return std::pair(IdxReg, SubRegs[0]);
3392 return std::pair(IdxBaseReg, SubRegs[
Offset]);
3395bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3401 LLT DstTy = MRI->getType(DstReg);
3402 LLT SrcTy = MRI->getType(SrcReg);
3404 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
3405 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
3406 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI);
3410 if (IdxRB->
getID() != AMDGPU::SGPRRegBankID)
3414 TRI.getRegClassForTypeOnBank(SrcTy, *SrcRB);
3416 TRI.getRegClassForTypeOnBank(DstTy, *DstRB);
3417 if (!SrcRC || !DstRC)
3419 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) ||
3420 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
3421 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI))
3430 *MRI, TRI, SrcRC, IdxReg,
3433 if (SrcRB->
getID() == AMDGPU::SGPRRegBankID) {
3437 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3440 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3442 .
addReg(SrcReg, {}, SubReg)
3444 MI.eraseFromParent();
3451 if (!STI.useVGPRIndexMode()) {
3452 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3454 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg)
3455 .
addReg(SrcReg, {}, SubReg)
3457 MI.eraseFromParent();
3461 const MCInstrDesc &GPRIDXDesc = TII.getIndirectGPRIDXPseudo(
3462 static_cast<unsigned>(TRI.getRegSizeInBits(*SrcRC)),
true);
3468 MI.eraseFromParent();
3473bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3480 LLT VecTy = MRI->getType(DstReg);
3481 LLT ValTy = MRI->getType(ValReg);
3483 unsigned ValSize =
static_cast<unsigned>(ValTy.
getSizeInBits());
3485 const RegisterBank *VecRB = RBI.getRegBank(VecReg, *MRI, TRI);
3486 const RegisterBank *ValRB = RBI.getRegBank(ValReg, *MRI, TRI);
3487 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI);
3493 if (IdxRB->
getID() != AMDGPU::SGPRRegBankID)
3497 TRI.getRegClassForTypeOnBank(VecTy, *VecRB);
3499 TRI.getRegClassForTypeOnBank(ValTy, *ValRB);
3501 if (!RBI.constrainGenericRegister(VecReg, *VecRC, *MRI) ||
3502 !RBI.constrainGenericRegister(DstReg, *VecRC, *MRI) ||
3503 !RBI.constrainGenericRegister(ValReg, *ValRC, *MRI) ||
3504 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI))
3507 if (VecRB->
getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3511 std::tie(IdxReg, SubReg) =
3514 const bool IndexMode = VecRB->
getID() == AMDGPU::VGPRRegBankID &&
3515 STI.useVGPRIndexMode();
3521 BuildMI(*BB, &
MI,
DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
3524 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3525 VecSize, ValSize, VecRB->
getID() == AMDGPU::SGPRRegBankID);
3530 MI.eraseFromParent();
3534 const MCInstrDesc &GPRIDXDesc = TII.getIndirectGPRIDXPseudo(
3535 static_cast<unsigned>(TRI.getRegSizeInBits(*VecRC)),
false);
3542 MI.eraseFromParent();
3549 return BufInfo->IsAsync;
3551 case Intrinsic::amdgcn_load_async_to_lds:
3552 case Intrinsic::amdgcn_global_load_async_lds:
3558bool AMDGPUInstructionSelector::selectBufferLoadLds(
3560 if (!Subtarget->hasVMemToLDSLoad())
3562 auto GetArg = [&](
unsigned Arg) -> MachineOperand & {
3566 unsigned Size =
static_cast<unsigned>(GetArg(*
Info.getSizeArg()).getImm());
3568 std::optional<unsigned> VIndexArg =
Info.getVIndexArg();
3569 const bool HasVIndex = VIndexArg.has_value();
3572 VIndex = GetArg(*VIndexArg).getReg();
3574 Register VOffset = GetArg(*
Info.getVOffsetArg()).getReg();
3575 std::optional<ValueAndVReg> MaybeVOffset =
3577 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3583 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3584 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3585 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3586 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3589 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3590 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3591 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3592 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3595 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3596 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3597 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3598 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3601 if (!Subtarget->hasLDSLoadB96_B128())
3604 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3605 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3606 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3607 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3610 if (!Subtarget->hasLDSLoadB96_B128())
3613 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3614 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3615 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3616 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3623 .
add(GetArg(*
Info.getLDSPtrArg()));
3627 if (HasVIndex && HasVOffset) {
3628 Register IdxReg = MRI->createVirtualRegister(TRI.getVGPR64Class());
3629 BuildMI(*
MBB, &*MIB,
DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg)
3636 }
else if (HasVIndex) {
3638 }
else if (HasVOffset) {
3642 MIB.
add(GetArg(
Info.RsrcArg));
3643 MIB.
add(GetArg(*
Info.getSOffsetArg()));
3644 MIB.
add(GetArg(*
Info.getImmOffsetArg()));
3646 int64_t Aux = GetArg(*
Info.getAuxArg()).getImm();
3655 MachineMemOperand *LoadMMO = *
MI.memoperands_begin();
3660 MachinePointerInfo StorePtrI = LoadPtrI;
3671 MachineMemOperand *StoreMMO =
3677 MI.eraseFromParent();
3690 if (
Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3696 return Def->getOperand(1).getReg();
3710 if (
Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3718 return Def->getOperand(1).getReg();
3721 if (ZextSrc &&
VT->signBitIsZero(ZextSrc))
3730AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(
Register Reg)
const {
3732 : matchZeroExtendFromS32(
Reg);
3738AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(
Register Reg)
const {
3740 : matchSignExtendFromS32(
Reg);
3744AMDGPUInstructionSelector::matchExtendFromS32OrS32(
Register Reg,
3745 bool IsSigned)
const {
3747 return matchSignExtendFromS32OrS32(
Reg);
3749 return matchZeroExtendFromS32OrS32(
Reg);
3759 if (
Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3766 return Def->getOperand(1).getReg();
3771bool AMDGPUInstructionSelector::selectGlobalLoadLds(
MachineInstr &
MI)
const{
3772 if (!Subtarget->hasVMemToLDSLoad())
3776 unsigned Size =
static_cast<unsigned>(
MI.getOperand(3).
getImm());
3783 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3786 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3789 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3792 if (!Subtarget->hasLDSLoadB96_B128())
3794 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3797 if (!Subtarget->hasLDSLoadB96_B128())
3799 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3806 .
add(
MI.getOperand(2));
3812 if (!isSGPR(Addr)) {
3814 if (isSGPR(AddrDef->Reg)) {
3815 Addr = AddrDef->Reg;
3816 }
else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3819 if (isSGPR(SAddr)) {
3820 Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
3821 if (
Register Off = matchZeroExtendFromS32(PtrBaseOffset)) {
3832 VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3844 MIB.
add(
MI.getOperand(4));
3846 unsigned Aux =
static_cast<unsigned>(
MI.getOperand(5).
getImm());
3850 MachineMemOperand *LoadMMO = *
MI.memoperands_begin();
3852 LoadPtrI.
Offset =
MI.getOperand(4).getImm();
3853 MachinePointerInfo StorePtrI = LoadPtrI;
3862 MachineMemOperand *StoreMMO =
3864 sizeof(int32_t),
Align(4));
3868 MI.eraseFromParent();
3873bool AMDGPUInstructionSelector::selectTensorLoadStore(
MachineInstr &
MI,
3875 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3877 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3881 const auto isAllZeros = [&](MachineOperand &Opnd) {
3882 const MachineInstr *
DefMI = MRI->getVRegDef(Opnd.getReg());
3891 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3892 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3899 .
add(
MI.getOperand(1))
3900 .
add(
MI.getOperand(2));
3902 if (NumGroups >= 4) {
3903 MIB.
add(
MI.getOperand(3))
3904 .
add(
MI.getOperand(4));
3908 .
add(
MI.getOperand(6));
3910 MI.eraseFromParent();
3914bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3916 unsigned OpcodeOpIdx =
3917 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3919 TII.get(
static_cast<unsigned>(
MI.getOperand(OpcodeOpIdx).getImm())));
3920 MI.removeOperand(OpcodeOpIdx);
3921 MI.addImplicitDefUseOperands(*
MI.getMF());
3928bool AMDGPUInstructionSelector::selectSMFMACIntrin(
MachineInstr &
MI)
const {
3931 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3932 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3934 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3935 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3937 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3938 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3940 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3941 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3943 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3944 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3946 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3947 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3949 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3950 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3952 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3953 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3955 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3956 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3958 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3959 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3961 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3962 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3964 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3965 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3967 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3968 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3970 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3971 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3973 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3974 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3976 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3977 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3979 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3980 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3982 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
3983 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
3985 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
3986 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
3988 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
3989 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
3991 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3992 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3994 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
3995 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
3997 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
3998 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
4000 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
4001 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
4003 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
4004 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
4006 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
4007 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
4009 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
4010 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
4012 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
4013 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
4019 auto VDst_In =
MI.getOperand(4);
4021 MI.setDesc(TII.get(
Opc));
4022 MI.removeOperand(4);
4023 MI.removeOperand(1);
4024 MI.addOperand(VDst_In);
4025 MI.addImplicitDefUseOperands(*
MI.getMF());
4026 const MCInstrDesc &MCID =
MI.getDesc();
4028 MI.getOperand(0).setIsEarlyClobber(
true);
4033bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
4035 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
4036 !Subtarget->hasPermlane16Swap())
4038 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
4039 !Subtarget->hasPermlane32Swap())
4042 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
4043 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
4044 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
4046 MI.removeOperand(2);
4047 MI.setDesc(TII.get(Opcode));
4050 MachineOperand &FI =
MI.getOperand(4);
4057bool AMDGPUInstructionSelector::selectWaveAddress(
MachineInstr &
MI)
const {
4060 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
4061 const bool IsVALU = DstRB->
getID() == AMDGPU::VGPRRegBankID;
4066 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg)
4067 .
addImm(Subtarget->getWavefrontSizeLog2())
4072 .
addImm(Subtarget->getWavefrontSizeLog2())
4077 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4078 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI))
4081 MI.eraseFromParent();
4085bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4095 const LLT DstTy = MRI->getType(DstReg);
4097 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
4099 TRI.getRegClassForSizeOnBank(DstSize, *DstRB);
4104 if (!Subtarget->supportsBPermute())
4108 if (Subtarget->supportsWaveWideBPermute()) {
4109 Register ShiftIdxReg = MRI->createVirtualRegister(DstRC);
4110 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), ShiftIdxReg)
4120 assert(Subtarget->isWave64());
4124 MRI->createVirtualRegister(TRI.getRegClass(AMDGPU::SReg_32RegClassID));
4125 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefValReg);
4127 Register UndefExecReg = MRI->createVirtualRegister(
4128 TRI.getRegClass(AMDGPU::SReg_64_XEXECRegClassID));
4129 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefExecReg);
4131 Register PoisonValReg = MRI->createVirtualRegister(DstRC);
4132 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_SET_INACTIVE_B32), PoisonValReg)
4140 Register ShiftIdxReg = MRI->createVirtualRegister(DstRC);
4141 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), ShiftIdxReg)
4145 Register PoisonIdxReg = MRI->createVirtualRegister(DstRC);
4146 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_SET_INACTIVE_B32), PoisonIdxReg)
4153 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(DstRC);
4155 PoisonUnshiftedIdxReg)
4163 Register SameSidePermReg = MRI->createVirtualRegister(DstRC);
4164 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::DS_BPERMUTE_B32), SameSidePermReg)
4169 Register SwappedValReg = MRI->createVirtualRegister(DstRC);
4170 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_PERMLANE64_B32), SwappedValReg)
4173 Register OppSidePermReg = MRI->createVirtualRegister(DstRC);
4174 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::DS_BPERMUTE_B32), OppSidePermReg)
4179 Register WWMSwapPermReg = MRI->createVirtualRegister(DstRC);
4180 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::STRICT_WWM), WWMSwapPermReg)
4187 Register ThreadIDReg = MRI->createVirtualRegister(DstRC);
4188 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_MBCNT_LO_U32_B32_e64), ThreadIDReg)
4192 Register XORReg = MRI->createVirtualRegister(DstRC);
4195 .
addReg(PoisonUnshiftedIdxReg);
4197 Register ANDReg = MRI->createVirtualRegister(DstRC);
4202 Register CompareReg = MRI->createVirtualRegister(
4203 TRI.getRegClass(AMDGPU::SReg_64_XEXECRegClassID));
4204 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), CompareReg)
4209 BuildMI(*
MBB,
MI,
DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
4217 MI.eraseFromParent();
4226 unsigned NumOpcodes = 0;
4239 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4250 for (
unsigned I = 0;
I < Src.size(); ++
I) {
4264 if (Src.size() == 3) {
4271 for (
unsigned I = 0;
I < Src.size(); ++
I) {
4272 if (Src[
I] ==
LHS) {
4282 Bits = SrcBits[Src.size()];
4288 switch (
MI->getOpcode()) {
4289 case TargetOpcode::G_AND:
4290 case TargetOpcode::G_OR:
4291 case TargetOpcode::G_XOR: {
4296 if (!getOperandBits(
LHS, LHSBits) ||
4297 !getOperandBits(
RHS, RHSBits)) {
4298 Src = std::move(Backup);
4320 uint8_t LHSBitsOrig = LHSBits;
4321 uint8_t RHSBitsOrig = RHSBits;
4325 NumOpcodes += LHSOp.first;
4326 LHSBits = LHSOp.second;
4333 NumOpcodes += RHSOp.first;
4334 RHSBits = RHSOp.second;
4338 auto dependsOnSlot = [](
uint8_t TT,
int Slot) ->
bool {
4339 if (Slot < 0 || Slot > 2)
4341 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4342 const int Shifts[3] = {4, 2, 1};
4343 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4349 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4356 for (
int I = 0;
I < (int)S.size();
I++) {
4357 if (Bits == SrcBitsConst[
I] && S[
I] ==
Op)
4359 if (IsNegationOp && Bits == (
uint8_t)~SrcBitsConst[
I] &&
4360 S[
I] == NegatedInner)
4371 for (
int I = 0;
I < (int)SrcAfterLHS.
size() &&
I < 3;
I++) {
4372 if (
I < (
int)Src.size() && Src[
I] != SrcAfterLHS[
I] &&
4373 dependsOnSlot(LHSBits,
I)) {
4382 if (!Stale && !RHSOp.first) {
4383 int Slot = findSlot(RHSBitsOrig,
RHS, SrcBeforeRecurse);
4385 (Slot >= (
int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4391 if (!Stale && !LHSOp.first) {
4392 int Slot = findSlot(LHSBitsOrig,
LHS, SrcBeforeRecurse);
4394 (Slot >= (
int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4399 Src = std::move(SrcBeforeRecurse);
4400 LHSBits = LHSBitsOrig;
4401 RHSBits = RHSBitsOrig;
4411 switch (
MI->getOpcode()) {
4412 case TargetOpcode::G_AND:
4413 TTbl = LHSBits & RHSBits;
4415 case TargetOpcode::G_OR:
4416 TTbl = LHSBits | RHSBits;
4418 case TargetOpcode::G_XOR:
4419 TTbl = LHSBits ^ RHSBits;
4425 return std::make_pair(NumOpcodes + 1, TTbl);
4428bool AMDGPUInstructionSelector::selectBITOP3(
MachineInstr &
MI)
const {
4429 if (!Subtarget->hasBitOp3Insts())
4433 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
4434 const bool IsVALU = DstRB->
getID() == AMDGPU::VGPRRegBankID;
4440 unsigned NumOpcodes;
4442 std::tie(NumOpcodes, TTbl) =
BitOp3_Op(DstReg, Src, *MRI);
4446 if (NumOpcodes < 2 || Src.empty())
4451 unsigned Size = MRI->getType(DstReg).getSizeInBits();
4452 assert((
Size == 16 ||
Size == 32) &&
"unexpected VALU logic op size");
4453 const bool IsB32 =
Size == 32;
4454 if (NumOpcodes == 2 && IsB32) {
4462 }
else if (NumOpcodes < 4) {
4469 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4470 if (!IsB32 && STI.hasTrue16BitInsts())
4471 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4472 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4473 unsigned CBL = STI.getConstantBusLimit(
Opc);
4477 for (
unsigned I = 0;
I < Src.size(); ++
I) {
4478 const RegisterBank *RB = RBI.getRegBank(Src[
I], *MRI, TRI);
4479 if (RB->
getID() != AMDGPU::SGPRRegBankID)
4485 Register NewReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4496 while (Src.size() < 3)
4497 Src.push_back(Src[0]);
4514 MI.eraseFromParent();
4519bool AMDGPUInstructionSelector::selectWriteRegister(
MachineInstr &
MI)
const {
4520 const MDString *RegStr =
4523 LLT Ty = MRI->getType(SrcReg);
4525 Register PhysReg = Subtarget->getTargetLowering()->getRegisterByName(
4530 "invalid register \"" + Twine(RegStr->
getString()) +
4531 "\" for llvm.write_register",
4532 Fn,
MI.getDebugLoc()));
4533 MI.eraseFromParent();
4537 if (!RBI.constrainGenericRegister(
4538 SrcReg, *TRI.getSGPRClassForBitWidth(Ty.
getSizeInBits()), *MRI))
4541 BuildMI(*
MI.getParent(),
MI,
MI.getDebugLoc(), TII.get(AMDGPU::COPY), PhysReg)
4543 MI.eraseFromParent();
4547bool AMDGPUInstructionSelector::selectStackRestore(
MachineInstr &
MI)
const {
4549 if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, *MRI))
4552 MachineInstr *
DefMI = MRI->getVRegDef(SrcReg);
4554 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4560 WaveAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
4563 .
addImm(Subtarget->getWavefrontSizeLog2())
4570 MI.eraseFromParent();
4576 if (!
I.isPreISelOpcode()) {
4578 return selectCOPY(
I);
4582 switch (
I.getOpcode()) {
4583 case TargetOpcode::G_AND:
4584 case TargetOpcode::G_OR:
4585 case TargetOpcode::G_XOR:
4586 if (selectBITOP3(
I))
4590 return selectG_AND_OR_XOR(
I);
4591 case TargetOpcode::G_ADD:
4592 case TargetOpcode::G_SUB:
4593 case TargetOpcode::G_PTR_ADD:
4596 return selectG_ADD_SUB(
I);
4597 case TargetOpcode::G_UADDO:
4598 case TargetOpcode::G_USUBO:
4599 case TargetOpcode::G_UADDE:
4600 case TargetOpcode::G_USUBE:
4601 return selectG_UADDO_USUBO_UADDE_USUBE(
I);
4602 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4603 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4604 return selectG_AMDGPU_MAD_64_32(
I);
4605 case TargetOpcode::G_INTTOPTR:
4606 case TargetOpcode::G_BITCAST:
4607 case TargetOpcode::G_PTRTOINT:
4608 case TargetOpcode::G_FREEZE:
4609 return selectCOPY(
I);
4610 case TargetOpcode::G_FNEG:
4613 return selectG_FNEG(
I);
4614 case TargetOpcode::G_FABS:
4617 return selectG_FABS(
I);
4618 case TargetOpcode::G_EXTRACT:
4619 return selectG_EXTRACT(
I);
4620 case TargetOpcode::G_MERGE_VALUES:
4621 case TargetOpcode::G_CONCAT_VECTORS:
4622 return selectG_MERGE_VALUES(
I);
4623 case TargetOpcode::G_UNMERGE_VALUES:
4624 return selectG_UNMERGE_VALUES(
I);
4625 case TargetOpcode::G_BUILD_VECTOR:
4626 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4627 return selectG_BUILD_VECTOR(
I);
4628 case TargetOpcode::G_IMPLICIT_DEF:
4629 return selectG_IMPLICIT_DEF(
I);
4630 case TargetOpcode::G_INSERT:
4631 return selectG_INSERT(
I);
4632 case TargetOpcode::G_INTRINSIC:
4633 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4634 return selectG_INTRINSIC(
I);
4635 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4636 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4637 return selectG_INTRINSIC_W_SIDE_EFFECTS(
I);
4638 case TargetOpcode::G_ICMP:
4639 case TargetOpcode::G_FCMP:
4640 if (selectG_ICMP_or_FCMP(
I))
4643 case TargetOpcode::G_LOAD:
4644 case TargetOpcode::G_ZEXTLOAD:
4645 case TargetOpcode::G_SEXTLOAD:
4646 case TargetOpcode::G_STORE:
4647 case TargetOpcode::G_ATOMIC_CMPXCHG:
4648 case TargetOpcode::G_ATOMICRMW_XCHG:
4649 case TargetOpcode::G_ATOMICRMW_ADD:
4650 case TargetOpcode::G_ATOMICRMW_SUB:
4651 case TargetOpcode::G_ATOMICRMW_AND:
4652 case TargetOpcode::G_ATOMICRMW_OR:
4653 case TargetOpcode::G_ATOMICRMW_XOR:
4654 case TargetOpcode::G_ATOMICRMW_MIN:
4655 case TargetOpcode::G_ATOMICRMW_MAX:
4656 case TargetOpcode::G_ATOMICRMW_UMIN:
4657 case TargetOpcode::G_ATOMICRMW_UMAX:
4658 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4659 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4660 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4661 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4662 case TargetOpcode::G_ATOMICRMW_FADD:
4663 case TargetOpcode::G_ATOMICRMW_FMIN:
4664 case TargetOpcode::G_ATOMICRMW_FMAX:
4665 return selectG_LOAD_STORE_ATOMICRMW(
I);
4666 case TargetOpcode::G_SELECT:
4667 return selectG_SELECT(
I);
4668 case TargetOpcode::G_TRUNC:
4669 return selectG_TRUNC(
I);
4670 case TargetOpcode::G_SEXT:
4671 case TargetOpcode::G_ZEXT:
4672 case TargetOpcode::G_ANYEXT:
4673 case TargetOpcode::G_SEXT_INREG:
4677 if (MRI->getType(
I.getOperand(1).getReg()) !=
LLT::scalar(1) &&
4680 return selectG_SZA_EXT(
I);
4681 case TargetOpcode::G_FPEXT:
4682 if (selectG_FPEXT(
I))
4685 case TargetOpcode::G_BRCOND:
4686 return selectG_BRCOND(
I);
4687 case TargetOpcode::G_GLOBAL_VALUE:
4688 return selectG_GLOBAL_VALUE(
I);
4689 case TargetOpcode::G_PTRMASK:
4690 return selectG_PTRMASK(
I);
4691 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4692 return selectG_EXTRACT_VECTOR_ELT(
I);
4693 case TargetOpcode::G_INSERT_VECTOR_ELT:
4694 return selectG_INSERT_VECTOR_ELT(
I);
4695 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4696 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4697 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4698 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4699 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4702 assert(Intr &&
"not an image intrinsic with image pseudo");
4703 return selectImageIntrinsic(
I, Intr);
4705 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4706 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4707 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4708 return selectBVHIntersectRayIntrinsic(
I);
4709 case AMDGPU::G_SBFX:
4710 case AMDGPU::G_UBFX:
4711 return selectG_SBFX_UBFX(
I);
4712 case AMDGPU::G_SI_CALL:
4713 I.setDesc(TII.get(AMDGPU::SI_CALL));
4715 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4716 return selectWaveAddress(
I);
4717 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4718 I.setDesc(TII.get(AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4721 case AMDGPU::G_STACKRESTORE:
4722 return selectStackRestore(
I);
4723 case TargetOpcode::G_WRITE_REGISTER:
4724 return selectWriteRegister(
I);
4726 return selectPHI(
I);
4727 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4728 return selectCOPY_SCC_VCC(
I);
4729 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4730 return selectCOPY_VCC_SCC(
I);
4731 case AMDGPU::G_AMDGPU_READANYLANE:
4732 return selectReadAnyLane(
I);
4733 case TargetOpcode::G_CONSTANT:
4734 case TargetOpcode::G_FCONSTANT:
4742AMDGPUInstructionSelector::selectVCSRC(
MachineOperand &Root)
const {
4749std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4750 Register Src,
bool IsCanonicalizing,
bool AllowAbs,
bool OpSel)
const {
4754 if (
MI->getOpcode() == AMDGPU::G_FNEG) {
4755 Src =
MI->getOperand(1).getReg();
4758 }
else if (
MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4763 if (
LHS &&
LHS->isZero()) {
4765 Src =
MI->getOperand(2).getReg();
4769 if (AllowAbs &&
MI->getOpcode() == AMDGPU::G_FABS) {
4770 Src =
MI->getOperand(1).getReg();
4777 return std::pair(Src, Mods);
4780std::pair<Register, unsigned>
4781AMDGPUInstructionSelector::selectVOP3PModsF32Impl(
Register Src)
const {
4783 std::tie(Src, Mods) = selectVOP3ModsImpl(Src);
4785 return std::pair(Src, Mods);
4788Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4790 bool ForceVGPR)
const {
4791 if ((Mods != 0 || ForceVGPR) &&
4792 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4799 TII.
get(AMDGPU::COPY), VGPRSrc)
4811AMDGPUInstructionSelector::widenSrcIfVGPR16(
Register Src,
4813 if (!Subtarget->useRealTrue16Insts() || MRI->getType(Src) !=
LLT::scalar(16))
4816 const RegisterBank *SrcRB = RBI.getRegBank(Src, *MRI, TRI);
4817 if (!SrcRB || SrcRB->
getID() != AMDGPU::VGPRRegBankID)
4820 MachineIRBuilder
B(*InsertPt);
4822 Register ImpDefReg = MRI->createVirtualRegister(&AMDGPU::VGPR_16RegClass);
4823 B.buildInstr(TargetOpcode::IMPLICIT_DEF).addDef(ImpDefReg);
4825 Register DstReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4826 B.buildInstr(AMDGPU::REG_SEQUENCE)
4829 .addImm(AMDGPU::lo16)
4831 .addImm(AMDGPU::hi16);
4840AMDGPUInstructionSelector::selectVSRC0(
MachineOperand &Root)
const {
4842 [=](MachineInstrBuilder &MIB) { MIB.
add(Root); }
4847AMDGPUInstructionSelector::selectVOP3Mods0(
MachineOperand &Root)
const {
4850 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.
getReg());
4853 [=](MachineInstrBuilder &MIB) {
4854 MIB.
addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4856 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); },
4857 [=](MachineInstrBuilder &MIB) { MIB.
addImm(0); },
4858 [=](MachineInstrBuilder &MIB) { MIB.
addImm(0); }
4863AMDGPUInstructionSelector::selectVOP3BMods0(
MachineOperand &Root)
const {
4866 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.
getReg(),
4871 [=](MachineInstrBuilder &MIB) {
4872 MIB.
addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4874 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); },
4875 [=](MachineInstrBuilder &MIB) { MIB.
addImm(0); },
4876 [=](MachineInstrBuilder &MIB) { MIB.
addImm(0); }
4881AMDGPUInstructionSelector::selectVOP3OMods(
MachineOperand &Root)
const {
4883 [=](MachineInstrBuilder &MIB) { MIB.
add(Root); },
4884 [=](MachineInstrBuilder &MIB) { MIB.
addImm(0); },
4885 [=](MachineInstrBuilder &MIB) { MIB.
addImm(0); }
4890AMDGPUInstructionSelector::selectVOP3Mods(
MachineOperand &Root)
const {
4893 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.
getReg());
4896 [=](MachineInstrBuilder &MIB) {
4897 MIB.
addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4899 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
4904AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4908 std::tie(Src, Mods) =
4909 selectVOP3ModsImpl(Root.
getReg(),
false);
4912 [=](MachineInstrBuilder &MIB) {
4913 MIB.
addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4915 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
4920AMDGPUInstructionSelector::selectVOP3BMods(
MachineOperand &Root)
const {
4923 std::tie(Src, Mods) =
4924 selectVOP3ModsImpl(Root.
getReg(),
true,
4928 [=](MachineInstrBuilder &MIB) {
4929 MIB.
addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
4931 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
4936AMDGPUInstructionSelector::selectVOP3NoMods(
MachineOperand &Root)
const {
4939 if (
Def->getOpcode() == AMDGPU::G_FNEG ||
Def->getOpcode() == AMDGPU::G_FABS)
4942 [=](MachineInstrBuilder &MIB) { MIB.
addReg(
Reg); },
4947AMDGPUInstructionSelector::selectVOP3NoOpSelMods(
MachineOperand &Root)
const {
4951 return selectVOP3NoMods(Root);
4975 if (
MI->getOpcode() != AMDGPU::G_TRUNC)
4978 uint64_t DstSize =
MRI.getType(
MI->getOperand(0).getReg()).getSizeInBits();
4979 uint64_t SrcSize =
MRI.getType(
MI->getOperand(1).getReg()).getSizeInBits();
4980 return DstSize * 2 == SrcSize;
4986 if (
MI->getOpcode() != AMDGPU::G_LSHR)
4990 std::optional<ValueAndVReg> ShiftAmt;
4993 uint64_t SrcSize =
MRI.getType(
MI->getOperand(1).getReg()).getSizeInBits();
4994 unsigned Shift =
static_cast<unsigned>(ShiftAmt->Value.getZExtValue());
4995 return Shift * 2 == SrcSize;
5003 if (
MI->getOpcode() != AMDGPU::G_SHL)
5007 std::optional<ValueAndVReg> ShiftAmt;
5010 uint64_t SrcSize =
MRI.getType(
MI->getOperand(1).getReg()).getSizeInBits();
5011 unsigned Shift =
static_cast<unsigned>(ShiftAmt->Value.getZExtValue());
5012 return Shift * 2 == SrcSize;
5020 if (
MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
5022 return MI->getNumOperands() == 3 &&
MI->getOperand(0).isDef() &&
5023 MI->getOperand(1).isDef() && !
MI->getOperand(2).isDef();
5031 if (
OpTy.isScalar())
5033 if (
OpTy.isVector() &&
OpTy.getNumElements() == 2)
5193static std::optional<std::pair<Register, SrcStatus>>
5198 unsigned Opc =
MI->getOpcode();
5202 case AMDGPU::G_BITCAST:
5203 return std::optional<std::pair<Register, SrcStatus>>(
5204 {
MI->getOperand(1).getReg(), Curr.second});
5206 if (
MI->getOperand(1).getReg().isPhysical())
5207 return std::nullopt;
5208 return std::optional<std::pair<Register, SrcStatus>>(
5209 {
MI->getOperand(1).getReg(), Curr.second});
5210 case AMDGPU::G_FNEG: {
5213 return std::nullopt;
5214 return std::optional<std::pair<Register, SrcStatus>>(
5215 {
MI->getOperand(1).getReg(), Stat});
5222 switch (Curr.second) {
5225 return std::optional<std::pair<Register, SrcStatus>>(
5228 if (Curr.first ==
MI->getOperand(0).getReg())
5229 return std::optional<std::pair<Register, SrcStatus>>(
5231 return std::optional<std::pair<Register, SrcStatus>>(
5243 return std::optional<std::pair<Register, SrcStatus>>(
5247 if (Curr.first ==
MI->getOperand(0).getReg())
5248 return std::optional<std::pair<Register, SrcStatus>>(
5250 return std::optional<std::pair<Register, SrcStatus>>(
5256 return std::optional<std::pair<Register, SrcStatus>>(
5261 return std::optional<std::pair<Register, SrcStatus>>(
5266 return std::optional<std::pair<Register, SrcStatus>>(
5271 return std::optional<std::pair<Register, SrcStatus>>(
5277 return std::nullopt;
5287 bool HasNeg =
false;
5289 bool HasOpsel =
true;
5294 unsigned Opc =
MI->getOpcode();
5296 if (
Opc == TargetOpcode::G_INTRINSIC) {
5299 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5326 while (
Depth <= MaxDepth && Curr.has_value()) {
5329 Statlist.push_back(Curr.value());
5336static std::pair<Register, SrcStatus>
5343 while (
Depth <= MaxDepth && Curr.has_value()) {
5349 LastSameOrNeg = Curr.value();
5354 return LastSameOrNeg;
5359 uint64_t Width1 =
MRI.getType(Reg1).getSizeInBits();
5360 uint64_t Width2 =
MRI.getType(Reg2).getSizeInBits();
5361 return Width1 == Width2;
5397 IsHalfState(HiStat);
5400std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5402 bool NoOpSel0)
const {
5407 return {RootReg, Mods};
5410 SearchOptions SO(RootReg, MRI);
5423 if (MRI.getType(RootReg).getSizeInBits() == 128) {
5425 return {Stat.first, Mods};
5430 MI->getNumOperands() != 3 || (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5432 return {Stat.first, Mods};
5438 if (StatlistHi.
empty()) {
5440 return {Stat.first, Mods};
5446 if (StatlistLo.
empty()) {
5448 return {Stat.first, Mods};
5451 for (
int I =
static_cast<int>(StatlistHi.
size()) - 1;
I >= 0;
I--) {
5452 for (
int J =
static_cast<int>(StatlistLo.
size()) - 1; J >= 0; J--) {
5453 if (StatlistHi[
I].first == StatlistLo[J].first &&
5455 StatlistHi[
I].first, RootReg, TII, MRI)) {
5457 updateMods(StatlistHi[
I].second, StatlistLo[J].second, Mods);
5460 return {StatlistHi[
I].first, NewMods};
5467 return {Stat.first, Mods};
5477 return RB->
getID() == RBNo;
5493 if (
checkRB(RootReg, AMDGPU::SGPRRegBankID, RBI,
MRI,
TRI) ||
5502 Register DstReg =
MRI.cloneVirtualRegister(RootReg);
5504 TII.get(AMDGPU::COPY), DstReg)
5512AMDGPUInstructionSelector::selectVOP3PRetHelper(
MachineOperand &Root,
5514 bool NoOpSel0)
const {
5518 std::tie(
Reg, Mods) =
5519 selectVOP3PModsImpl(Root.
getReg(), MRI, IsDOT, NoOpSel0);
5522 [=, &Root, &MRI](MachineInstrBuilder &MIB) {
5526 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
5531AMDGPUInstructionSelector::selectVOP3PMods(
MachineOperand &Root)
const {
5533 return selectVOP3PRetHelper(Root);
5537AMDGPUInstructionSelector::selectVOP3PModsPkFmaOpSel0(
5540 LLT Ty = MRI.getType(Root.
getReg());
5541 bool NoOpSel0 = Subtarget->hasPkF32InvalidOpSel() && Ty.
isFixedVector() &&
5543 return selectVOP3PRetHelper(Root,
false, NoOpSel0);
5547AMDGPUInstructionSelector::selectVOP3PModsPkAddMulOpSel0(
5550 LLT Ty = MRI.getType(Root.
getReg());
5551 if (Subtarget->hasPkF32InvalidOpSel() && Ty.
isFixedVector() &&
5554 return std::nullopt;
5555 return selectVOP3PRetHelper(Root);
5559AMDGPUInstructionSelector::selectVOP3PModsDOT(
MachineOperand &Root)
const {
5561 return selectVOP3PRetHelper(Root,
true);
5565AMDGPUInstructionSelector::selectVOP3PNoModsDOT(
MachineOperand &Root)
const {
5569 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.
getReg(), MRI,
true );
5573 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); }}};
5577AMDGPUInstructionSelector::selectVOP3PModsF32(
MachineOperand &Root)
const {
5580 std::tie(Src, Mods) = selectVOP3PModsF32Impl(Root.
getReg());
5583 [=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); },
5584 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
5589AMDGPUInstructionSelector::selectVOP3PNoModsF32(
MachineOperand &Root)
const {
5592 std::tie(Src, Mods) = selectVOP3PModsF32Impl(Root.
getReg());
5596 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); }}};
5600AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5603 "expected i1 value");
5609 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
5617 switch (Elts.
size()) {
5619 DstRegClass = &AMDGPU::VReg_256RegClass;
5622 DstRegClass = &AMDGPU::VReg_128RegClass;
5625 DstRegClass = &AMDGPU::VReg_64RegClass;
5632 auto MIB =
B.buildInstr(AMDGPU::REG_SEQUENCE)
5633 .addDef(
MRI.createVirtualRegister(DstRegClass));
5634 for (
unsigned i = 0; i < Elts.
size(); ++i) {
5645 if (ModOpcode == TargetOpcode::G_FNEG) {
5649 for (
auto El : Elts) {
5655 if (Elts.size() != NegAbsElts.
size()) {
5664 assert(ModOpcode == TargetOpcode::G_FABS);
5672AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(
MachineOperand &Root)
const {
5681 MachineInstr *ElF32 = MRI->getVRegDef(BV->
getSourceReg(0));
5682 unsigned ModOpcode = (ElF32->
getOpcode() == AMDGPU::G_FNEG)
5699 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); },
5700 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }}};
5704AMDGPUInstructionSelector::selectWMMAModsF16Neg(
MachineOperand &Root)
const {
5726 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); },
5727 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }}};
5731AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(
MachineOperand &Root)
const {
5739 MachineInstr *ElV2F16 = MRI->getVRegDef(CV->
getSourceReg(0));
5741 unsigned ModOpcode = (ElV2F16->
getOpcode() == AMDGPU::G_FNEG)
5759 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); },
5760 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }}};
5764AMDGPUInstructionSelector::selectWMMAVISrc(
MachineOperand &Root)
const {
5765 std::optional<FPValueAndVReg> FPValReg;
5767 if (TII.isInlineConstant(FPValReg->Value)) {
5768 return {{[=](MachineInstrBuilder &MIB) {
5769 MIB.
addImm(FPValReg->Value.bitcastToAPInt().getSExtValue());
5779 if (TII.isInlineConstant(ICst)) {
5789AMDGPUInstructionSelector::selectSWMMACIndex8(
MachineOperand &Root)
const {
5795 std::optional<ValueAndVReg> ShiftAmt;
5797 MRI->getType(ShiftSrc).getSizeInBits() == 32 &&
5798 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5799 Key =
static_cast<unsigned>(ShiftAmt->Value.getZExtValue() / 8);
5804 [=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); },
5805 [=](MachineInstrBuilder &MIB) { MIB.
addImm(
Key); }
5810AMDGPUInstructionSelector::selectSWMMACIndex16(
MachineOperand &Root)
const {
5817 std::optional<ValueAndVReg> ShiftAmt;
5819 MRI->getType(ShiftSrc).getSizeInBits() == 32 &&
5820 ShiftAmt->Value.getZExtValue() == 16) {
5826 [=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); },
5827 [=](MachineInstrBuilder &MIB) { MIB.
addImm(
Key); }
5832AMDGPUInstructionSelector::selectSWMMACIndex32(
MachineOperand &Root)
const {
5839 S32 = matchAnyExtendFromS32(Src);
5843 if (
Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5848 Src =
Def->getOperand(2).getReg();
5855 [=](MachineInstrBuilder &MIB) { MIB.
addReg(Src); },
5856 [=](MachineInstrBuilder &MIB) { MIB.
addImm(
Key); }
5861AMDGPUInstructionSelector::selectVOP3OpSelMods(
MachineOperand &Root)
const {
5864 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.
getReg());
5867 if (!Subtarget->useRealTrue16Insts() &&
5868 MRI->getType(Root.
getReg()).getSizeInBits() == 16 &&
5875 [=](MachineInstrBuilder &MIB) {
5876 MIB.
addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB));
5878 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
5884AMDGPUInstructionSelector::selectVINTERPMods(
MachineOperand &Root)
const {
5887 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.
getReg(),
5893 [=](MachineInstrBuilder &MIB) {
5895 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB,
true));
5897 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); },
5902AMDGPUInstructionSelector::selectVINTERPModsHi(
MachineOperand &Root)
const {
5905 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.
getReg(),
5911 [=](MachineInstrBuilder &MIB) {
5913 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB,
true));
5915 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); },
5922bool AMDGPUInstructionSelector::selectScaleOffset(
MachineOperand &Root,
5925 bool IsSMEM)
const {
5926 if (!Subtarget->hasScaleOffset())
5930 MachineMemOperand *MMO = *
MI.memoperands_begin();
5938 bool IsExtended = OffsetReg && OffsetReg !=
Offset;
5943 OffsetReg =
Def->Reg;
5958 m_BinOp(IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5962 (
Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5963 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5964 (IsSigned &&
Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5965 VT->signBitIsZero(
Mul->getOperand(2).getReg()))) &&
5978 if (!IsExtended && MRI->getType(
Offset).getSizeInBits() == 64) {
5979 KnownBits
Known =
VT->getKnownBits(Op0);
5980 if (
Known.countMaxActiveBits() + Bits > 32)
5983 }
else if (IsExtended) {
5984 const MachineInstr *Scale = MRI->getVRegDef(OffsetReg);
5986 VT->computeNumSignBits(Op0) <= Bits)
5989 VT->getKnownBits(Op0).countMinLeadingZeros() < Bits)
5997bool AMDGPUInstructionSelector::selectSmrdOffset(
MachineOperand &Root,
6001 bool *ScaleOffset)
const {
6008 getAddrModeInfo(*
MI, *MRI, AddrInfo);
6010 if (AddrInfo.
empty())
6013 const GEPInfo &GEPI = AddrInfo[0];
6014 std::optional<int64_t> EncodedImm;
6017 *ScaleOffset =
false;
6022 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
6023 AddrInfo.
size() > 1) {
6024 const GEPInfo &GEPI2 = AddrInfo[1];
6025 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
6026 Register OffsetReg = GEPI2.SgprParts[1];
6028 *ScaleOffset = selectScaleOffset(
6029 Root, OffsetReg,
false ,
true );
6030 OffsetReg = matchZeroExtendFromS32OrS32(OffsetReg);
6032 Base = GEPI2.SgprParts[0];
6033 *SOffset = OffsetReg;
6042 auto SKnown =
VT->getKnownBits(*SOffset);
6043 if (*
Offset + SKnown.getMinValue().getSExtValue() < 0)
6055 if (
Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
6056 Base = GEPI.SgprParts[0];
6062 if (SOffset && GEPI.SgprParts.size() == 1 &&
isUInt<32>(GEPI.Imm) &&
6068 Base = GEPI.SgprParts[0];
6069 *SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
6070 BuildMI(*
MBB,
MI,
MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), *SOffset)
6075 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
6076 Register OffsetReg = GEPI.SgprParts[1];
6078 *ScaleOffset = selectScaleOffset(Root, OffsetReg,
false ,
6080 OffsetReg = matchZeroExtendFromS32OrS32(OffsetReg);
6082 Base = GEPI.SgprParts[0];
6083 *SOffset = OffsetReg;
6092AMDGPUInstructionSelector::selectSmrdImm(
MachineOperand &Root)
const {
6095 if (!selectSmrdOffset(Root,
Base,
nullptr, &
Offset,
6097 return std::nullopt;
6099 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(
Base); },
6100 [=](MachineInstrBuilder &MIB) { MIB.
addImm(
Offset); }}};
6104AMDGPUInstructionSelector::selectSmrdImm32(
MachineOperand &Root)
const {
6106 getAddrModeInfo(*Root.
getParent(), *MRI, AddrInfo);
6108 if (AddrInfo.
empty() || AddrInfo[0].SgprParts.size() != 1)
6109 return std::nullopt;
6111 const GEPInfo &GEPInfo = AddrInfo[0];
6112 Register PtrReg = GEPInfo.SgprParts[0];
6113 std::optional<int64_t> EncodedImm =
6116 return std::nullopt;
6119 [=](MachineInstrBuilder &MIB) { MIB.
addReg(PtrReg); },
6120 [=](MachineInstrBuilder &MIB) { MIB.
addImm(*EncodedImm); }
6125AMDGPUInstructionSelector::selectSmrdSgpr(
MachineOperand &Root)
const {
6128 if (!selectSmrdOffset(Root,
Base, &SOffset,
nullptr,
6130 return std::nullopt;
6133 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(
Base); },
6134 [=](MachineInstrBuilder &MIB) { MIB.
addReg(SOffset); },
6135 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPol); }}};
6139AMDGPUInstructionSelector::selectSmrdSgprImm(
MachineOperand &Root)
const {
6143 if (!selectSmrdOffset(Root,
Base, &SOffset, &
Offset, &ScaleOffset))
6144 return std::nullopt;
6147 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(
Base); },
6148 [=](MachineInstrBuilder &MIB) { MIB.
addReg(SOffset); },
6150 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPol); }}};
6153std::pair<Register, int64_t> AMDGPUInstructionSelector::selectFlatOffsetImpl(
6159 if (!STI.hasFlatInstOffsets())
6163 int64_t ConstOffset;
6165 std::tie(PtrBase, ConstOffset, IsInBounds) =
6166 getPtrBaseWithConstantOffset(Root.
getReg(), *MRI);
6172 if (ConstOffset == 0 ||
6174 !isFlatScratchBaseLegal(Root.
getReg())) ||
6178 unsigned AddrSpace = (*
MI->memoperands_begin())->getAddrSpace();
6179 if (!TII.isLegalFLATOffset(ConstOffset, AddrSpace, FlatVariant))
6182 return std::pair(PtrBase, ConstOffset);
6186AMDGPUInstructionSelector::selectFlatOffset(
MachineOperand &Root)
const {
6190 [=](MachineInstrBuilder &MIB) { MIB.
addReg(PtrWithOffset.first); },
6191 [=](MachineInstrBuilder &MIB) { MIB.
addImm(PtrWithOffset.second); },
6196AMDGPUInstructionSelector::selectGlobalOffset(
MachineOperand &Root)
const {
6197 auto PtrWithOffset =
6201 [=](MachineInstrBuilder &MIB) { MIB.
addReg(PtrWithOffset.first); },
6202 [=](MachineInstrBuilder &MIB) { MIB.
addImm(PtrWithOffset.second); },
6207AMDGPUInstructionSelector::selectScratchOffset(
MachineOperand &Root)
const {
6208 auto PtrWithOffset =
6212 [=](MachineInstrBuilder &MIB) { MIB.
addReg(PtrWithOffset.first); },
6213 [=](MachineInstrBuilder &MIB) { MIB.
addImm(PtrWithOffset.second); },
6219AMDGPUInstructionSelector::selectGlobalSAddr(
MachineOperand &Root,
6221 bool NeedIOffset)
const {
6224 int64_t ConstOffset;
6225 int64_t ImmOffset = 0;
6229 std::tie(PtrBase, ConstOffset, std::ignore) =
6230 getPtrBaseWithConstantOffset(Addr, *MRI);
6232 if (ConstOffset != 0) {
6237 ImmOffset = ConstOffset;
6240 if (isSGPR(PtrBaseDef->Reg)) {
6246 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6248 std::tie(SplitImmOffset, RemainderOffset) =
6253 if (Subtarget->hasSignedGVSOffset() ?
isInt<32>(RemainderOffset)
6258 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6260 BuildMI(*
MBB,
MI,
MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
6262 .
addImm(RemainderOffset);
6266 [=](MachineInstrBuilder &MIB) { MIB.
addReg(PtrBase); },
6267 [=](MachineInstrBuilder &MIB) {
6270 [=](MachineInstrBuilder &MIB) { MIB.
addImm(SplitImmOffset); },
6271 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPolBits); },
6274 [=](MachineInstrBuilder &MIB) { MIB.
addReg(PtrBase); },
6275 [=](MachineInstrBuilder &MIB) {
6278 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPolBits); },
6287 unsigned NumLiterals =
6288 !TII.isInlineConstant(APInt(32,
Lo_32(ConstOffset))) +
6289 !TII.isInlineConstant(APInt(32,
Hi_32(ConstOffset)));
6290 if (STI.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
6291 return std::nullopt;
6298 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6303 if (isSGPR(SAddr)) {
6304 Register PtrBaseOffset = AddrDef->MI->getOperand(2).getReg();
6308 bool ScaleOffset = selectScaleOffset(Root, PtrBaseOffset,
6309 Subtarget->hasSignedGVSOffset());
6310 if (
Register VOffset = matchExtendFromS32OrS32(
6311 PtrBaseOffset, Subtarget->hasSignedGVSOffset())) {
6313 return {{[=](MachineInstrBuilder &MIB) {
6316 [=](MachineInstrBuilder &MIB) {
6319 [=](MachineInstrBuilder &MIB) {
6322 [=](MachineInstrBuilder &MIB) {
6326 return {{[=](MachineInstrBuilder &MIB) {
6329 [=](MachineInstrBuilder &MIB) {
6332 [=](MachineInstrBuilder &MIB) {
6342 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6343 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(AddrDef->Reg))
6344 return std::nullopt;
6350 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6352 BuildMI(*
MBB,
MI,
MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), VOffset)
6357 [=](MachineInstrBuilder &MIB) { MIB.
addReg(AddrDef->Reg); },
6358 [=](MachineInstrBuilder &MIB) { MIB.
addReg(VOffset); },
6359 [=](MachineInstrBuilder &MIB) { MIB.
addImm(ImmOffset); },
6360 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPolBits); }
6363 [=](MachineInstrBuilder &MIB) { MIB.
addReg(AddrDef->Reg); },
6364 [=](MachineInstrBuilder &MIB) { MIB.
addReg(VOffset); },
6365 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPolBits); }
6370AMDGPUInstructionSelector::selectGlobalSAddr(
MachineOperand &Root)
const {
6371 return selectGlobalSAddr(Root, 0);
6375AMDGPUInstructionSelector::selectGlobalSAddrCPol(
MachineOperand &Root)
const {
6381 return selectGlobalSAddr(Root,
static_cast<unsigned>(PassedCPol));
6385AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(
MachineOperand &Root)
const {
6391 return selectGlobalSAddr(Root,
static_cast<unsigned>(PassedCPol));
6395AMDGPUInstructionSelector::selectGlobalSAddrGLC(
MachineOperand &Root)
const {
6400AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6407 return selectGlobalSAddr(Root,
static_cast<unsigned>(PassedCPol),
false);
6411AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6418 return selectGlobalSAddr(Root,
static_cast<unsigned>(PassedCPol),
false);
6422AMDGPUInstructionSelector::selectScratchSAddr(
MachineOperand &Root)
const {
6425 int64_t ConstOffset;
6426 int64_t ImmOffset = 0;
6430 std::tie(PtrBase, ConstOffset, std::ignore) =
6431 getPtrBaseWithConstantOffset(Addr, *MRI);
6433 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6437 ImmOffset = ConstOffset;
6441 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6442 int FI = AddrDef->MI->getOperand(1).
getIndex();
6445 [=](MachineInstrBuilder &MIB) { MIB.
addImm(ImmOffset); }
6451 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6452 Register LHS = AddrDef->MI->getOperand(1).getReg();
6453 Register RHS = AddrDef->MI->getOperand(2).getReg();
6457 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6458 isSGPR(RHSDef->Reg)) {
6459 int FI = LHSDef->MI->getOperand(1).getIndex();
6463 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
6465 BuildMI(*BB, &
I,
DL, TII.get(AMDGPU::S_ADD_I32), SAddr)
6473 return std::nullopt;
6476 [=](MachineInstrBuilder &MIB) { MIB.
addReg(SAddr); },
6477 [=](MachineInstrBuilder &MIB) { MIB.
addImm(ImmOffset); }
6482bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6484 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6490 auto VKnown =
VT->getKnownBits(VAddr);
6493 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6495 return (VMax & 3) + (
SMax & 3) >= 4;
6499AMDGPUInstructionSelector::selectScratchSVAddr(
MachineOperand &Root)
const {
6502 int64_t ConstOffset;
6503 int64_t ImmOffset = 0;
6507 std::tie(PtrBase, ConstOffset, std::ignore) =
6508 getPtrBaseWithConstantOffset(Addr, *MRI);
6511 if (ConstOffset != 0 &&
6515 ImmOffset = ConstOffset;
6519 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6520 return std::nullopt;
6522 Register RHS = AddrDef->MI->getOperand(2).getReg();
6523 if (RBI.getRegBank(
RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6524 return std::nullopt;
6526 Register LHS = AddrDef->MI->getOperand(1).getReg();
6529 if (OrigAddr != Addr) {
6530 if (!isFlatScratchBaseLegalSVImm(OrigAddr))
6531 return std::nullopt;
6533 if (!isFlatScratchBaseLegalSV(OrigAddr))
6534 return std::nullopt;
6537 if (checkFlatScratchSVSSwizzleBug(
RHS,
LHS, ImmOffset))
6538 return std::nullopt;
6540 unsigned CPol = selectScaleOffset(Root,
RHS,
true )
6544 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6545 int FI = LHSDef->MI->getOperand(1).getIndex();
6547 [=](MachineInstrBuilder &MIB) { MIB.
addReg(
RHS); },
6549 [=](MachineInstrBuilder &MIB) { MIB.
addImm(ImmOffset); },
6550 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPol); }
6559 return std::nullopt;
6562 [=](MachineInstrBuilder &MIB) { MIB.
addReg(
RHS); },
6563 [=](MachineInstrBuilder &MIB) { MIB.
addReg(
LHS); },
6564 [=](MachineInstrBuilder &MIB) { MIB.
addImm(ImmOffset); },
6565 [=](MachineInstrBuilder &MIB) { MIB.
addImm(CPol); }
6570AMDGPUInstructionSelector::selectMUBUFScratchOffen(
MachineOperand &Root)
const {
6574 const SIMachineFunctionInfo *
Info =
MF->getInfo<SIMachineFunctionInfo>();
6579 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6584 BuildMI(*
MBB,
MI,
MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
6588 return {{[=](MachineInstrBuilder &MIB) {
6591 [=](MachineInstrBuilder &MIB) {
6594 [=](MachineInstrBuilder &MIB) {
6599 [=](MachineInstrBuilder &MIB) {
6608 std::optional<int> FI;
6612 int64_t ConstOffset;
6613 std::tie(PtrBase, ConstOffset, std::ignore) =
6614 getPtrBaseWithConstantOffset(VAddr, *MRI);
6616 if (ConstOffset != 0) {
6617 if (TII.isLegalMUBUFImmOffset(ConstOffset) &&
6618 (!STI.privateMemoryResourceIsRangeChecked() ||
6619 VT->signBitIsZero(PtrBase))) {
6630 return {{[=](MachineInstrBuilder &MIB) {
6633 [=](MachineInstrBuilder &MIB) {
6639 [=](MachineInstrBuilder &MIB) {
6644 [=](MachineInstrBuilder &MIB) {
6649bool AMDGPUInstructionSelector::isDSOffsetLegal(
Register Base,
6654 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6659 return VT->signBitIsZero(
Base);
6662bool AMDGPUInstructionSelector::isDSOffset2Legal(
Register Base, int64_t Offset0,
6664 unsigned Size)
const {
6665 if (Offset0 %
Size != 0 || Offset1 %
Size != 0)
6670 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6675 return VT->signBitIsZero(
Base);
6680 return Addr->
getOpcode() == TargetOpcode::G_OR ||
6681 (Addr->
getOpcode() == TargetOpcode::G_PTR_ADD &&
6688bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(
Register Addr)
const {
6696 if (STI.hasSignedScratchOffsets())
6702 if (AddrMI->
getOpcode() == TargetOpcode::G_PTR_ADD) {
6703 std::optional<ValueAndVReg> RhsValReg =
6709 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6710 RhsValReg->Value.getSExtValue() > -0x40000000)
6714 return VT->signBitIsZero(
LHS);
6719bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(
Register Addr)
const {
6727 if (STI.hasSignedScratchOffsets())
6732 return VT->signBitIsZero(
RHS) &&
VT->signBitIsZero(
LHS);
6737bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6741 if (STI.hasSignedScratchOffsets())
6746 std::optional<DefinitionAndSourceRegister> BaseDef =
6748 std::optional<ValueAndVReg> RHSOffset =
6758 (RHSOffset->Value.getSExtValue() < 0 &&
6759 RHSOffset->Value.getSExtValue() > -0x40000000)))
6762 Register LHS = BaseDef->MI->getOperand(1).getReg();
6763 Register RHS = BaseDef->MI->getOperand(2).getReg();
6764 return VT->signBitIsZero(
RHS) &&
VT->signBitIsZero(
LHS);
6767bool AMDGPUInstructionSelector::isUnneededShiftMask(
const MachineInstr &
MI,
6768 unsigned ShAmtBits)
const {
6769 assert(
MI.getOpcode() == TargetOpcode::G_AND);
6773 std::optional<ValueAndVReg>
RHS =
6778 if (
RHS->
Value.countr_one() >= ShAmtBits)
6781 const APInt &LHSKnownZeros =
VT->getKnownZeroes(
MI.getOperand(1).getReg());
6782 return (LHSKnownZeros |
RHS->
Value).countr_one() >= ShAmtBits;
6786AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6789 const SIMachineFunctionInfo *
Info =
MF->getInfo<SIMachineFunctionInfo>();
6791 std::optional<DefinitionAndSourceRegister>
Def =
6793 assert(Def &&
"this shouldn't be an optional result");
6798 [=](MachineInstrBuilder &MIB) {
6801 [=](MachineInstrBuilder &MIB) {
6804 [=](MachineInstrBuilder &MIB) { MIB.
addImm(0); }
6815 if (!TII.isLegalMUBUFImmOffset(
Offset))
6823 [=](MachineInstrBuilder &MIB) {
6826 [=](MachineInstrBuilder &MIB) {
6834 !TII.isLegalMUBUFImmOffset(
Offset))
6838 [=](MachineInstrBuilder &MIB) {
6841 [=](MachineInstrBuilder &MIB) {
6848std::pair<Register, int64_t>
6849AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(
6851 int64_t ConstAddr = 0;
6855 std::tie(PtrBase,
Offset, std::ignore) =
6856 getPtrBaseWithConstantOffset(Root.
getReg(), *MRI);
6859 if (isDSOffsetLegal(PtrBase,
Offset)) {
6861 return std::pair(PtrBase,
Offset);
6870 return std::pair(Root.
getReg(), 0);
6874AMDGPUInstructionSelector::selectDS1Addr1Offset(
MachineOperand &Root)
const {
6877 std::tie(
Reg,
Offset) = selectDS1Addr1OffsetImpl(Root);
6879 [=](MachineInstrBuilder &MIB) { MIB.
addReg(
Reg); },
6885AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(
MachineOperand &Root)
const {
6886 return selectDSReadWrite2(Root, 4);
6890AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(
MachineOperand &Root)
const {
6891 return selectDSReadWrite2(Root, 8);
6895AMDGPUInstructionSelector::selectDSReadWrite2(
MachineOperand &Root,
6896 unsigned Size)
const {
6901 [=](MachineInstrBuilder &MIB) { MIB.
addReg(
Reg); },
6903 [=](MachineInstrBuilder &MIB) { MIB.
addImm(
Offset+1); }
6907std::pair<Register, int64_t>
6908AMDGPUInstructionSelector::selectDSReadWrite2Impl(
MachineOperand &Root,
6909 unsigned Size)
const {
6910 int64_t ConstAddr = 0;
6914 std::tie(PtrBase,
Offset, std::ignore) =
6915 getPtrBaseWithConstantOffset(Root.
getReg(), *MRI);
6918 int64_t OffsetValue0 =
Offset;
6920 if (isDSOffset2Legal(PtrBase, OffsetValue0, OffsetValue1,
Size)) {
6922 return std::pair(PtrBase, OffsetValue0 /
Size);
6931 return std::pair(Root.
getReg(), 0);
6939std::tuple<Register, int64_t, bool>
6940AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6943 if (RootI->
getOpcode() != TargetOpcode::G_PTR_ADD)
6944 return {Root, 0,
false};
6947 std::optional<ValueAndVReg> MaybeOffset =
6950 return {Root, 0,
false};
6965 Register RSrc2 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6966 Register RSrc3 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6967 Register RSrcHi =
MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6968 Register RSrc =
MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
6970 B.buildInstr(AMDGPU::S_MOV_B32)
6973 B.buildInstr(AMDGPU::S_MOV_B32)
6980 B.buildInstr(AMDGPU::REG_SEQUENCE)
6983 .addImm(AMDGPU::sub0)
6985 .addImm(AMDGPU::sub1);
6989 RSrcLo =
MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6990 B.buildInstr(AMDGPU::S_MOV_B64)
6995 B.buildInstr(AMDGPU::REG_SEQUENCE)
6998 .addImm(AMDGPU::sub0_sub1)
7000 .addImm(AMDGPU::sub2_sub3);
7007 uint64_t DefaultFormat =
TII.getDefaultRsrcDataFormat();
7016 uint64_t DefaultFormat =
TII.getDefaultRsrcDataFormat();
7023AMDGPUInstructionSelector::MUBUFAddressData
7024AMDGPUInstructionSelector::parseMUBUFAddress(
Register Src)
const {
7025 MUBUFAddressData
Data;
7031 std::tie(PtrBase,
Offset, std::ignore) =
7032 getPtrBaseWithConstantOffset(Src, *MRI);
7038 if (MachineInstr *InputAdd
7040 Data.N2 = InputAdd->getOperand(1).getReg();
7041 Data.N3 = InputAdd->getOperand(2).getReg();
7056bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr)
const {
7062 const RegisterBank *N0Bank = RBI.getRegBank(Addr.N0, *MRI, TRI);
7063 return N0Bank->
getID() == AMDGPU::VGPRRegBankID;
7069void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
7071 if (TII.isLegalMUBUFImmOffset(ImmOffset))
7075 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7076 B.buildInstr(AMDGPU::S_MOV_B32)
7082bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
7087 if (!STI.hasAddr64() || STI.useFlatForGlobal())
7090 MUBUFAddressData AddrData = parseMUBUFAddress(Root.
getReg());
7091 if (!shouldUseAddr64(AddrData))
7097 Offset = AddrData.Offset;
7103 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7105 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7118 }
else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7129 splitIllegalMUBUFOffset(
B, SOffset,
Offset);
7133bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
7138 if (STI.useFlatForGlobal())
7141 MUBUFAddressData AddrData = parseMUBUFAddress(Root.
getReg());
7142 if (shouldUseAddr64(AddrData))
7148 Offset = AddrData.Offset;
7154 splitIllegalMUBUFOffset(
B, SOffset,
Offset);
7159AMDGPUInstructionSelector::selectMUBUFAddr64(
MachineOperand &Root)
const {
7165 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset,
Offset))
7171 [=](MachineInstrBuilder &MIB) {
7174 [=](MachineInstrBuilder &MIB) {
7177 [=](MachineInstrBuilder &MIB) {
7180 else if (STI.hasRestrictedSOffset())
7181 MIB.
addReg(AMDGPU::SGPR_NULL);
7185 [=](MachineInstrBuilder &MIB) {
7195AMDGPUInstructionSelector::selectMUBUFOffset(
MachineOperand &Root)
const {
7200 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset,
Offset))
7204 [=](MachineInstrBuilder &MIB) {
7207 [=](MachineInstrBuilder &MIB) {
7210 else if (STI.hasRestrictedSOffset())
7211 MIB.
addReg(AMDGPU::SGPR_NULL);
7223AMDGPUInstructionSelector::selectBUFSOffset(
MachineOperand &Root)
const {
7228 SOffset = AMDGPU::SGPR_NULL;
7230 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(SOffset); }}};
7234static std::optional<uint64_t>
7238 if (!OffsetVal || !
isInt<32>(*OffsetVal))
7239 return std::nullopt;
7240 return Lo_32(*OffsetVal);
7244AMDGPUInstructionSelector::selectSMRDBufferImm(
MachineOperand &Root)
const {
7245 std::optional<uint64_t> OffsetVal =
7250 std::optional<int64_t> EncodedImm =
7255 return {{ [=](MachineInstrBuilder &MIB) { MIB.
addImm(*EncodedImm); } }};
7259AMDGPUInstructionSelector::selectSMRDBufferImm32(
MachineOperand &Root)
const {
7266 std::optional<int64_t> EncodedImm =
7271 return {{ [=](MachineInstrBuilder &MIB) { MIB.
addImm(*EncodedImm); } }};
7275AMDGPUInstructionSelector::selectSMRDBufferSgprImm(
MachineOperand &Root)
const {
7283 return std::nullopt;
7285 std::optional<int64_t> EncodedOffset =
7288 return std::nullopt;
7290 assert(MRI->getType(SOffset).getSizeInBits() == 32);
7291 return {{[=](MachineInstrBuilder &MIB) { MIB.
addReg(SOffset); },
7292 [=](MachineInstrBuilder &MIB) { MIB.
addImm(*EncodedOffset); }}};
7295std::pair<Register, unsigned>
7296AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(
MachineOperand &Root,
7297 bool &Matched)
const {
7302 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.
getReg());
7312 const auto CheckAbsNeg = [&]() {
7317 std::tie(Src, ModsTmp) = selectVOP3ModsImpl(Src);
7347 if (MRI->getType(Src).isFixedVector(2, 16))
7360AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7365 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7370 [=](MachineInstrBuilder &MIB) { MIB.
addReg(widenSrcIfVGPR16(Src, MIB)); },
7371 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
7376AMDGPUInstructionSelector::selectVOP3PMadMixMods(
MachineOperand &Root)
const {
7380 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7383 [=](MachineInstrBuilder &MIB) { MIB.
addReg(widenSrcIfVGPR16(Src, MIB)); },
7384 [=](MachineInstrBuilder &MIB) { MIB.
addImm(Mods); }
7389AMDGPUInstructionSelector::selectVOP3PMadMixModsExtNeg(
7394 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7399 [=](MachineInstrBuilder &MIB) { MIB.
addReg(widenSrcIfVGPR16(Src, MIB)); },
7400 [=](MachineInstrBuilder &MIB) {
7407AMDGPUInstructionSelector::selectVOP3PMadMixModsNeg(
7412 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7415 [=](MachineInstrBuilder &MIB) { MIB.
addReg(widenSrcIfVGPR16(Src, MIB)); },
7416 [=](MachineInstrBuilder &MIB) {
7422bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7426 Register CCReg =
I.getOperand(0).getReg();
7431 BuildMI(*
MBB, &
I,
DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7432 .
addImm(
I.getOperand(2).getImm());
7436 I.eraseFromParent();
7437 return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass,
7441bool AMDGPUInstructionSelector::selectSGetBarrierState(
7445 const MachineOperand &BarOp =
I.getOperand(2);
7446 std::optional<int64_t> BarValImm =
7450 auto CopyMIB =
BuildMI(*
MBB, &
I,
DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
7454 MachineInstrBuilder MIB;
7455 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7456 : AMDGPU::S_GET_BARRIER_STATE_M0;
7459 auto DstReg =
I.getOperand(0).getReg();
7461 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
7462 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
7468 I.eraseFromParent();
7474static std::optional<uint32_t>
7479 return *BarValImm & 0x3F;
7482 return std::nullopt;
7486 if (HasInlineConst) {
7490 case Intrinsic::amdgcn_s_barrier_join:
7491 return AMDGPU::S_BARRIER_JOIN_IMM;
7492 case Intrinsic::amdgcn_s_wakeup_barrier:
7493 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7494 case Intrinsic::amdgcn_s_get_named_barrier_state:
7495 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7501 case Intrinsic::amdgcn_s_barrier_join:
7502 return AMDGPU::S_BARRIER_JOIN_M0;
7503 case Intrinsic::amdgcn_s_wakeup_barrier:
7504 return AMDGPU::S_WAKEUP_BARRIER_M0;
7505 case Intrinsic::amdgcn_s_get_named_barrier_state:
7506 return AMDGPU::S_GET_BARRIER_STATE_M0;
7511bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7515 const MachineOperand &BarOp =
I.getOperand(1);
7516 const MachineOperand &CntOp =
I.getOperand(2);
7520 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7521 std::optional<int64_t> CntImm =
7523 if (CntImm && *CntImm == 0) {
7525 BuildMI(*
MBB, &
I,
DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_IMM))
7527 I.eraseFromParent();
7534 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7543 Register TmpReg2 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7549 Register TmpReg3 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7550 constexpr unsigned ShAmt = 16;
7556 Register TmpReg4 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7566 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7567 ? AMDGPU::S_BARRIER_INIT_M0
7568 : AMDGPU::S_BARRIER_SIGNAL_M0;
7569 MachineInstrBuilder MIB;
7572 I.eraseFromParent();
7576bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7580 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7583 std::optional<uint32_t> BarValImm =
7588 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
7594 auto CopyMIB =
BuildMI(*
MBB, &
I,
DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
7599 MachineInstrBuilder MIB;
7603 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7604 auto DstReg =
I.getOperand(0).getReg();
7606 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
7607 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
7615 I.eraseFromParent();
7622 assert(
MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7623 "Expected G_CONSTANT");
7624 MIB.
addImm(-
MI.getOperand(1).getCImm()->getSExtValue());
7630 const MachineOperand &
Op =
MI.getOperand(1);
7631 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7632 MIB.
addImm(
Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7635void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7637 assert(
MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7638 "Expected G_CONSTANT");
7639 MIB.
addImm(
MI.getOperand(1).getCImm()->getValue().countTrailingOnes());
7647 const MachineOperand &
Op =
MI.getOperand(OpIdx);
7665 MIB.
addImm(
MI.getOperand(OpIdx).getImm() != 0);
7671 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7675void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7677 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7682void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7684 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7685 MIB.
addImm((
MI.getOperand(OpIdx).getImm() & 0x1)
7690void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7692 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7697void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7699 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7700 MIB.
addImm((
MI.getOperand(OpIdx).getImm() & 0x2)
7705void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7707 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7712void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7714 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7719void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7721 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7726void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7728 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7729 MIB.
addImm((
MI.getOperand(OpIdx).getImm() & 0x2)
7737 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7738 MIB.
addImm(
MI.getOperand(OpIdx).getImm() &
7746 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7747 const bool Swizzle =
MI.getOperand(OpIdx).getImm() &
7753void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7755 assert(OpIdx >= 0 &&
"expected to match an immediate operand");
7756 const uint32_t Cpol =
MI.getOperand(OpIdx).getImm() &
7765 const APFloat &APF =
MI.getOperand(1).getFPImm()->getValueAPF();
7767 assert(ExpVal != INT_MIN);
7778 MIB.
addImm((
MI.getOperand(OpIdx).getImm() + 3) % 4);
7785 if (
MI.getOperand(OpIdx).getImm())
7787 MIB.
addImm((int64_t)Mods);
7794 if (
MI.getOperand(OpIdx).getImm())
7796 MIB.
addImm((int64_t)Mods);
7802 unsigned Val =
static_cast<unsigned>(
MI.getOperand(OpIdx).
getImm());
7810 MIB.
addImm((int64_t)Mods);
7818 if (!Subtarget->hasSafeCUPrefetch())
7824void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7826 int64_t Val =
MI.getOperand(OpIdx).getImm();
7835bool AMDGPUInstructionSelector::isInlineImmediate(
const APInt &
Imm)
const {
7836 return TII.isInlineConstant(
Imm);
7839bool AMDGPUInstructionSelector::isInlineImmediate(
const APFloat &
Imm)
const {
7840 return TII.isInlineConstant(
Imm);
MachineInstrBuilder MachineInstrBuilder & DefMI
static unsigned getIntrinsicID(const SDNode *N)
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test if the MI is shift left with half bits, such as reg0:2n =G_SHL reg1:2n, CONST(n)
static bool isNoUnsignedWrap(MachineInstr *Addr)
static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, const SIInstrInfo &TII, Register BasePtr)
unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID)
static Register getLegalRegBank(Register NewReg, Register RootReg, MachineInstr &Use, const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const SIInstrInfo &TII)
static bool checkRB(Register Reg, unsigned int RBNo, const AMDGPURegisterBankInfo &RBI, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI)
static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods)
static bool isTruncHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test if the MI is truncating to half, such as reg0:n = G_TRUNC reg1:2n
static Register getWaveAddress(const MachineInstr *Def)
static int getV_CMPOpcode(CmpInst::Predicate P, uint64_t Size, const GCNSubtarget &ST)
static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In, Register &Out)
static std::pair< unsigned, uint8_t > BitOp3_Op(Register R, SmallVectorImpl< Register > &Src, const MachineRegisterInfo &MRI)
static TypeClass isVectorOfTwoOrScalar(Register Reg, const MachineRegisterInfo &MRI)
static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI, MachineBasicBlock *MBB)
static bool isExtractLoElt(MachineRegisterInfo &MRI, Register In, Register &Out)
static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE, bool &IsTexFail)
static void addZeroImm(MachineInstrBuilder &MIB)
static unsigned gwsIntrinToOpcode(unsigned IntrID)
static bool isConstant(const MachineInstr &MI)
static bool isSameBitWidth(Register Reg1, Register Reg2, const MachineRegisterInfo &MRI)
static Register buildRegSequence(SmallVectorImpl< Register > &Elts, MachineInstr *InsertPt, MachineRegisterInfo &MRI)
static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI, uint32_t FormatLo, uint32_t FormatHi, Register BasePtr)
Return a resource descriptor for use with an arbitrary 64-bit pointer.
static bool isAsyncLDSDMA(Intrinsic::ID Intr)
static std::optional< uint32_t > getConstantNamedBarrierNumber(const MachineOperand &BarOp, const MachineRegisterInfo &MRI)
Return the barrier ID for BarOp if it's statically known, mapping poison to 0 and masking off the hig...
static std::pair< Register, unsigned > computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, Register IdxReg, unsigned EltSize, GISelValueTracking &ValueTracking)
Return the register to use for the index value, and the subregister to use for the indirectly accesse...
static bool shouldUseAndMask(uint64_t Size, unsigned &Mask)
static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64)
static std::pair< Register, SrcStatus > getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO, int MaxDepth=3)
static Register stripCopy(Register Reg, MachineRegisterInfo &MRI)
static std::optional< std::pair< Register, SrcStatus > > calcNextStatus(std::pair< Register, SrcStatus > Curr, const MachineRegisterInfo &MRI)
static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI)
static std::optional< uint64_t > getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI)
Get an immediate that must be 32-bits, and treated as zero extended.
static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg, Register RootReg, const SIInstrInfo &TII, const MachineRegisterInfo &MRI)
static SmallVector< std::pair< Register, SrcStatus > > getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO, int MaxDepth=3)
static bool isUnmergeHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test function, if the MI is reg0:n, reg1:n = G_UNMERGE_VALUES reg2:2n
static SrcStatus getNegStatus(Register Reg, SrcStatus S, const MachineRegisterInfo &MRI)
static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI)
static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, const SIInstrInfo &TII, Register BasePtr)
static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI)
Test if the MI is logic shift right with half bits, such as reg0:2n =G_LSHR reg1:2n,...
static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods, SmallVectorImpl< Register > &Elts, Register &Src, MachineInstr *InsertPt, MachineRegisterInfo &MRI)
This file declares the targeting of the InstructionSelector class for AMDGPU.
AMDGPU Register Bank Select
This file declares the targeting of the RegisterBankInfo class for AMDGPU.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool isAllZeros(StringRef Arr)
Return true if the array is empty or all zeros.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static std::vector< std::pair< int, unsigned > > Swizzle(std::vector< std::pair< int, unsigned > > Src, R600InstrInfo::BankSwizzle Swz)
MachineRegisterInfo & MRI
This is used to control valid status that current MI supports.
bool checkOptions(SrcStatus Stat) const
SearchOptions(Register Reg, const MachineRegisterInfo &MRI)
AMDGPUInstructionSelector(const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
static const char * getName()
bool select(MachineInstr &I) override
Select the (possibly generic) instruction I to only use target-specific opcodes.
void setupMF(MachineFunction &MF, GISelValueTracking *VT, CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) override
Setup per-MF executor state.
uint32_t getLDSSize() const
LLVM_READONLY int getExactLog2Abs() const
Class for arbitrary precision integers.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
int64_t getSExtValue() const
Get sign extended value.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
DILocation * get() const
Get the underlying DILocation.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void checkSubtargetFeatures(const Function &F) const
Diagnose inconsistent subtarget features before attempting to codegen function F.
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
virtual void setupMF(MachineFunction &mf, GISelValueTracking *vt, CodeGenCoverage *covinfo=nullptr, ProfileSummaryInfo *psi=nullptr, BlockFrequencyInfo *bfi=nullptr)
Setup per-MF executor state.
CodeGenCoverage * CoverageInfo
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents a G_UNMERGE_VALUES.
unsigned getNumDefs() const
Returns the number of def registers.
Register getSourceReg() const
Get the unmerge source register.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
static LLT integer(unsigned SizeInBits)
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
TypeSize getValue() const
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
bool hasSuperClassEq(const MCRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
unsigned getID() const
getID() - Return the register class ID number.
bool hasSubClassEq(const MCRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
LLVM_ABI StringRef getString() const
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setReturnAddressIsTaken(bool s)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Helper class to build MachineInstr.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
LocationSize getSize() const
Return the size in bytes of the memory reference.
unsigned getAddrSpace() const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
const Value * getValue() const
Return the base address of the memory access.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
const ConstantInt * getCImm() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
bool isEarlyClobber() const
Register getReg() const
getReg - Returns the register number.
bool isInternalRead() const
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Analysis providing profile information.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
const SDValue & getOperand(unsigned i) const
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
static unsigned getDSShaderTypeValue(const MachineFunction &MF)
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
static bool isGenericOpcode(unsigned Opc)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM_ABI Value(Type *Ty, unsigned scid)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
LLVM_READONLY const MIMGG16MappingInfo * getMIMGG16MappingInfo(unsigned G)
std::optional< int64_t > getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, int64_t ByteOffset)
bool isGFX12Plus(const MCSubtargetInfo &STI)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc, bool IndexedSamp)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST)
LLVM_READONLY int32_t getGlobalSaddrOp(uint32_t Opcode)
bool isGFX13Plus(const MCSubtargetInfo &STI)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isGFX10Plus(const MCSubtargetInfo &STI)
std::optional< int64_t > getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset, bool IsBuffer, bool HasSOffset)
unsigned getMIROperandIdx(const MachineInstr &MI, unsigned Arg)
Convert IR argument index Arg of the intrinsic called by MI, a G_INTRINSIC* instruction,...
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
unsigned getBarrierMemberCountBitMask(const IsaVersion &Version)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
Intrinsic::ID getIntrinsicID(const MachineInstr &I)
Return the intrinsic ID for opcodes with the G_AMDGPU_INTRIN_ prefix.
std::pair< Register, unsigned > getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg, GISelValueTracking *ValueTracking=nullptr, bool CheckNUW=false)
Returns base register and constant offset.
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
IndexMode
ARM Index Modes.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue)
Matches a constant equal to RequestedValue.
GInstrBind< GBuildVector > m_GBuildVector(GBuildVector *&Inst)
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
UnaryOp_match< SrcTy, TargetOpcode::COPY > m_Copy(SrcTy &&Src)
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_XOR, true > m_GXor(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_SEXT > m_GSExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_FPEXT > m_GFPExt(const SrcTy &Src)
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
ConstantMatch< APInt > m_ICst(APInt &Cst)
SpecificConstantMatch m_AllOnesInt()
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
ImplicitDefMatch m_GImplicitDef()
GInstrBind< GConcatVectors > m_GConcatVectors(GConcatVectors *&Inst)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
GInstrBind< GUnmerge > m_GUnmerge(GUnmerge *&Inst)
Instruction binders for ops with no operand-form matcher (constant-immediate or variadic-source ops).
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
SpecificRegisterMatch m_SpecificReg(Register RequestedReg)
Matches a register only if it is equal to RequestedReg.
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
GFrameIndexMatch m_GFrameIndex(int &FI)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_BITCAST > m_GBitcast(const SrcTy &Src)
bind_ty< MachineInstr * > m_MInstr(MachineInstr *&MI)
UnaryOp_match< SrcTy, TargetOpcode::G_FNEG > m_GFNeg(const SrcTy &Src)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
UnaryOp_match< SrcTy, TargetOpcode::G_FABS > m_GFabs(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ANYEXT > m_GAnyExt(const SrcTy &Src)
ShuffleVectorMatch< Src1Ty, Src2Ty > m_GShuffleVector(const Src1Ty &Src1, const Src2Ty &Src2, ArrayRef< int > &Mask)
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, TargetOpcode::G_MUL, true > m_GMul(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
@ Lo16
Fixup[15:0] <- Target + Addend.
@ Hi16
Fixup[15:0] <- (Target + Addend + 0x8000) >> 16.
NodeAddr< DefNode * > Def
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
LLVM_ABI bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
LLVM_ABI Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
PointerUnion< const TargetRegisterClass *, const RegisterBank * > RegClassOrRegBank
Convenient type to represent either a register class or a register bank.
LLVM_ABI const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
@ Known
Known to have no common set bits.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
@ Load
The value being inserted comes from a load (InsertElement only).
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI std::optional< ValueAndVReg > getAnyConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true, bool LookThroughAnyExt=false)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT or G_FCONST...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
LLVM_ABI std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
LLVM_ABI Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
constexpr RegState getUndefRegState(bool B)
MCRegisterClass TargetRegisterClass
The argument layout of a buffer intrinsic.
unsigned AtomicNoRetBaseOpcode
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
int64_t Offset
Offset - This is an offset from the base Value*.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.