34#include "llvm/IR/IntrinsicsAMDGPU.h"
42#define DEBUG_TYPE "si-instr-info"
44#define GET_INSTRINFO_CTOR_DTOR
45#include "AMDGPUGenInstrInfo.inc"
48#define GET_ImageDimIntrinsicTable_IMPL
49#define GET_RsrcIntrinsics_IMPL
50#define GET_BufferIntrinsicsTable_IMPL
51#define GET_GFX1250BlockingCyclesTable_DECL
52#define GET_GFX1250BlockingCyclesTable_IMPL
59#include "AMDGPUGenSearchableTables.inc"
74 unsigned N =
Node->getNumOperands();
75 while (
N &&
Node->getOperand(
N - 1).getValueType() == MVT::Glue)
87 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0,
OpName);
88 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1,
OpName);
90 if (Op0Idx == -1 && Op1Idx == -1)
94 if ((Op0Idx == -1 && Op1Idx != -1) ||
95 (Op1Idx == -1 && Op0Idx != -1))
116 return !
MI.memoperands_empty() &&
118 return MMO->isLoad() && MMO->isInvariant();
127static std::pair<unsigned, unsigned>
129 const unsigned Flags =
SrcOp.getTargetFlags();
137 const unsigned GotFlags =
139 return {GotFlags, GotFlags};
145 std::optional<unsigned> LoFlags =
147 std::optional<unsigned> HiFlags =
149 if (!LoFlags || !HiFlags)
151 return {*LoFlags, *HiFlags};
169 if (!
MI.hasImplicitDef() &&
170 MI.getNumImplicitOperands() ==
MI.getDesc().implicit_uses().size() &&
171 !
MI.mayRaiseFPException())
180 if (!
MI.getNumOperands() || !
MI.getOperand(0).isReg())
195 if (
MI.isNotDuplicable() ||
MI.mayStore() ||
MI.mayRaiseFPException() ||
196 MI.hasUnmodeledSideEffects())
201 if (
MI.isInlineAsm())
205 if (
MI.mayLoad() && !
MI.isDereferenceableInvariantLoad())
220 if (Reg.isPhysical()) {
225 if (!
MRI.isConstantPhysReg(Reg))
236 if (MO.isDef() && Reg != DefReg)
246 case AMDGPU::V_SUBREV_U16_e32:
247 case AMDGPU::V_SUBREV_U16_e64:
249 case AMDGPU::V_SUBREV_U32_e32:
250 case AMDGPU::V_SUBREV_U32_e64:
252 case AMDGPU::V_SUBREV_CO_U32_e32:
253 case AMDGPU::V_SUBREV_CO_U32_e64:
255 case AMDGPU::V_SUBBREV_U32_e32:
256 case AMDGPU::V_SUBBREV_U32_e64:
259 case AMDGPU::V_ASHRREV_I16_e32:
260 case AMDGPU::V_ASHRREV_I16_e64:
261 case AMDGPU::V_ASHRREV_I32_e32:
262 case AMDGPU::V_ASHRREV_I32_e64:
263 case AMDGPU::V_ASHRREV_I64_e64:
264 case AMDGPU::V_LSHLREV_B16_e32:
265 case AMDGPU::V_LSHLREV_B16_e64:
266 case AMDGPU::V_LSHLREV_B32_e32:
267 case AMDGPU::V_LSHLREV_B32_e64:
268 case AMDGPU::V_LSHLREV_B64_e64:
269 case AMDGPU::V_LSHRREV_B16_e32:
270 case AMDGPU::V_LSHRREV_B16_e64:
271 case AMDGPU::V_LSHRREV_B32_e32:
272 case AMDGPU::V_LSHRREV_B32_e64:
273 case AMDGPU::V_LSHRREV_B64_e64:
274 return !ST.hasGFX11Insts();
281bool SIInstrInfo::resultDependsOnExec(
const MachineInstr &
MI)
const {
285 if (
MI.isConvergent())
313 if (
MI.getOpcode() == AMDGPU::SI_IF_BREAK)
318 for (
auto Op :
MI.uses()) {
319 if (
Op.isReg() &&
Op.getReg().isVirtual() &&
320 RI.isSGPRClass(
MRI.getRegClass(
Op.getReg()))) {
333 while (FromCycle && !(ToCycle && CI->
contains(FromCycle, ToCycle))) {
353 int64_t &Offset1)
const {
361 if (!
get(Opc0).mayLoad() || !
get(Opc1).mayLoad())
365 if (!
get(Opc0).getNumDefs() || !
get(Opc1).getNumDefs())
381 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
382 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
383 if (Offset0Idx == -1 || Offset1Idx == -1)
390 Offset0Idx -=
get(Opc0).NumDefs;
391 Offset1Idx -=
get(Opc1).NumDefs;
421 if (!Load0Offset || !Load1Offset)
438 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
439 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
441 if (OffIdx0 == -1 || OffIdx1 == -1)
447 OffIdx0 -=
get(Opc0).NumDefs;
448 OffIdx1 -=
get(Opc1).NumDefs;
467 case AMDGPU::DS_READ2ST64_B32:
468 case AMDGPU::DS_READ2ST64_B64:
469 case AMDGPU::DS_WRITE2ST64_B32:
470 case AMDGPU::DS_WRITE2ST64_B64:
484 OffsetIsScalable =
false;
501 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
503 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::data0);
504 if (
Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
517 unsigned Offset0 = Offset0Op->
getImm() & 0xff;
518 unsigned Offset1 = Offset1Op->
getImm() & 0xff;
519 if (Offset0 + 1 != Offset1)
530 int Data0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::data0);
531 EltSize = RI.getRegSizeInBits(*
getOpRegClass(LdSt, Data0Idx)) / 8;
538 Offset = EltSize * Offset0;
540 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
541 if (DataOpIdx == -1) {
542 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::data0);
544 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::data1);
560 if (BaseOp && !BaseOp->
isFI())
568 if (SOffset->
isReg())
574 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
576 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
585 isMIMG(LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
586 int SRsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RsrcOpName);
588 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
589 if (VAddr0Idx >= 0) {
591 for (
int I = VAddr0Idx;
I < SRsrcIdx; ++
I)
598 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
613 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
630 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
632 DataOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
642 if (!LdStIdx->isGPRIdx())
644 BaseOp = &LdStIdx->getIdxOp();
665 if (BaseOps1.
front()->isIdenticalTo(*BaseOps2.
front()))
673 if (MO1->getAddrSpace() != MO2->getAddrSpace())
676 const auto *Base1 = MO1->getValue();
677 const auto *Base2 = MO2->getValue();
678 if (!Base1 || !Base2)
686 return Base1 == Base2;
697 *BaseOps2.
front()->getParent(), BaseOps2);
701 int64_t Offset1,
bool OffsetIsScalable1,
703 int64_t Offset2,
bool OffsetIsScalable2,
704 unsigned ClusterSize,
705 unsigned NumBytes)
const {
715 if (!BaseOps1.
empty()) {
717 MaxMemoryClusterDWords =
735 const unsigned LoadSize = NumBytes / ClusterSize;
736 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;
737 return NumDWords <= MaxMemoryClusterDWords;
751 int64_t Offset0, int64_t Offset1,
752 unsigned NumLoads)
const {
753 assert(Offset1 > Offset0 &&
754 "Second offset should be larger than first offset!");
759 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
766 const char *
Msg =
"illegal VGPR to SGPR copy") {
785 assert((
TII.getSubtarget().hasMAIInsts() &&
786 !
TII.getSubtarget().hasGFX90AInsts()) &&
787 "Expected GFX908 subtarget.");
790 AMDGPU::AGPR_32RegClass.
contains(SrcReg)) &&
791 "Source register of the copy should be either an SGPR or an AGPR.");
794 "Destination register of the copy should be an AGPR.");
803 for (
auto Def =
MI,
E =
MBB.begin(); Def !=
E; ) {
806 if (!Def->modifiesRegister(SrcReg, &RI))
809 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
810 Def->getOperand(0).getReg() != SrcReg)
817 bool SafeToPropagate =
true;
820 for (
auto I = Def;
I !=
MI && SafeToPropagate; ++
I)
821 if (
I->modifiesRegister(DefOp.
getReg(), &RI))
822 SafeToPropagate =
false;
824 if (!SafeToPropagate)
827 for (
auto I = Def;
I !=
MI; ++
I)
828 I->clearRegisterKills(DefOp.
getReg(), &RI);
836 if (ImpUseSuperReg) {
837 Builder.addReg(ImpUseSuperReg,
845 RS.enterBasicBlockEnd(
MBB);
846 RS.backward(std::next(
MI));
855 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
858 assert(
MBB.getParent()->getRegInfo().isReserved(Tmp) &&
859 "VGPR used for an intermediate copy should have been reserved.");
864 Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass,
MI,
874 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
875 if (AMDGPU::AGPR_32RegClass.
contains(SrcReg)) {
876 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
883 if (ImpUseSuperReg) {
884 UseBuilder.
addReg(ImpUseSuperReg,
901 for (
unsigned Idx = 0; Idx < BaseIndices.
size(); ++Idx) {
902 int16_t SubIdx = BaseIndices[Idx];
903 Register DestSubReg = RI.getSubReg(DestReg, SubIdx);
904 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
905 assert(DestSubReg && SrcSubReg &&
"Failed to find subregs!");
906 unsigned Opcode = AMDGPU::S_MOV_B32;
909 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
910 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
911 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.
size())) {
915 DestSubReg = RI.getSubReg(DestReg, SubIdx);
916 SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
917 assert(DestSubReg && SrcSubReg &&
"Failed to find subregs!");
918 Opcode = AMDGPU::S_MOV_B64;
933 assert(FirstMI && LastMI);
938 LastMI->addRegisterKilled(SrcReg, &RI);
944 Register SrcReg,
bool KillSrc,
bool RenamableDest,
945 bool RenamableSrc)
const {
947 unsigned Size = RI.getRegSizeInBits(*RC);
949 unsigned SrcSize = RI.getRegSizeInBits(*SrcRC);
954 if (ST.getCLOpts().fix_16_bit_physreg_copies) {
955 if (((
Size == 16) != (SrcSize == 16))) {
957 assert(ST.useRealTrue16Insts());
959 MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16);
962 if (DestReg == SrcReg) {
968 RC = RI.getPhysRegBaseClass(DestReg);
969 Size = RI.getRegSizeInBits(*RC);
970 SrcRC = RI.getPhysRegBaseClass(SrcReg);
971 SrcSize = RI.getRegSizeInBits(*SrcRC);
975 if (RC == &AMDGPU::VGPR_32RegClass) {
977 AMDGPU::SReg_32RegClass.
contains(SrcReg) ||
978 AMDGPU::AGPR_32RegClass.
contains(SrcReg));
979 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
980 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
986 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
987 RC == &AMDGPU::SReg_32RegClass) {
988 if (SrcReg == AMDGPU::SCC) {
995 if (!AMDGPU::SReg_32RegClass.
contains(SrcReg)) {
996 if (DestReg == AMDGPU::VCC_LO) {
1014 if (RC == &AMDGPU::SReg_64RegClass) {
1015 if (SrcReg == AMDGPU::SCC) {
1022 if (!AMDGPU::SReg_64_EncodableRegClass.
contains(SrcReg)) {
1023 if (DestReg == AMDGPU::VCC) {
1041 if (DestReg == AMDGPU::SCC) {
1044 if (AMDGPU::SReg_64RegClass.
contains(SrcReg)) {
1048 assert(ST.hasScalarCompareEq64());
1062 if (RC == &AMDGPU::AGPR_32RegClass) {
1063 if (AMDGPU::VGPR_32RegClass.
contains(SrcReg) ||
1064 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) {
1070 if (AMDGPU::AGPR_32RegClass.
contains(SrcReg) && ST.hasGFX90AInsts()) {
1079 const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
1086 AMDGPU::SReg_LO16RegClass.
contains(SrcReg) ||
1087 AMDGPU::AGPR_LO16RegClass.
contains(SrcReg));
1089 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
1090 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
1091 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
1092 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
1095 MCRegister NewDestReg = RI.get32BitRegister(DestReg);
1096 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg);
1109 if (IsAGPRDst || IsAGPRSrc) {
1110 if (!DstLow || !SrcLow) {
1112 "Cannot use hi16 subreg with an AGPR!");
1119 if (ST.useRealTrue16Insts()) {
1125 if (AMDGPU::VGPR_16_Lo128RegClass.
contains(DestReg) &&
1126 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.
contains(SrcReg))) {
1138 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1139 if (!DstLow || !SrcLow) {
1141 "Cannot use hi16 subreg on VI!");
1167 unsigned SrcOp = 1) {
1171 return DstOpRC && SrcOpRC && DstOpRC->
contains(Dst) &&
1175 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {
1176 if (ST.hasVMovB64Inst() &&
1177 CanCopyWith(AMDGPU::V_MOV_B64_e32, DestReg, SrcReg)) {
1182 if (ST.hasPkMovB32() &&
1183 CanCopyWith(AMDGPU::V_PK_MOV_B32, DestReg, SrcReg, 2)) {
1199 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
1200 if (RI.isSGPRClass(RC)) {
1201 if (!RI.isSGPRClass(SrcRC)) {
1205 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
1211 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1212 unsigned WideOpcode = AMDGPU::INSTRUCTION_LIST_END;
1213 if (RI.isAGPRClass(RC)) {
1214 if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC))
1215 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1216 else if (RI.hasVGPRs(SrcRC) ||
1217 (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC)))
1218 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1220 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1221 }
else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) {
1222 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1223 }
else if (RI.isVGPRClass(RC)) {
1224 if (ST.hasVMovB64Inst())
1225 WideOpcode = AMDGPU::V_MOV_B64_e32;
1226 else if (ST.hasPkMovB32())
1227 WideOpcode = AMDGPU::V_PK_MOV_B32;
1231 if (WideOpcode != AMDGPU::INSTRUCTION_LIST_END) {
1233 unsigned SrcOp = WideOpcode == AMDGPU::V_PK_MOV_B32 ? 2 : 1;
1240 const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
1241 const bool CanKillSuperReg = KillSrc && !Overlap;
1248 std::unique_ptr<RegScavenger> RS;
1249 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1250 RS = std::make_unique<RegScavenger>();
1254 for (
unsigned Idx{}; Idx < SubIndices.
size();) {
1255 unsigned NumRegs = 1;
1256 unsigned ThisOpcode = Opcode;
1258 Forward ? SubIndices[Idx] : SubIndices[SubIndices.
size() - Idx - 1];
1260 if (WideDstRC && WideSrcRC && Idx + 1 < SubIndices.
size()) {
1261 unsigned Channel = RI.getChannelFromSubReg(SubIdx);
1265 unsigned WideSubIdx = RI.getSubRegFromChannel(Channel, 2);
1266 Register WideDst = RI.getSubReg(DestReg, WideSubIdx);
1267 Register WideSrc = RI.getSubReg(SrcReg, WideSubIdx);
1269 if (WideDst && WideSrc && WideDstRC->
contains(WideDst) &&
1270 WideSrcRC->contains(WideSrc)) {
1271 SubIdx = WideSubIdx;
1273 ThisOpcode = WideOpcode;
1277 Register DestSubReg = RI.getSubReg(DestReg, SubIdx);
1278 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
1279 assert(DestSubReg && SrcSubReg &&
"Failed to find subregs!");
1282 bool UseKill = CanKillSuperReg && Idx == SubIndices.
size();
1284 if (ThisOpcode == AMDGPU::INSTRUCTION_LIST_END) {
1287 *RS, Overlap, ImpUseSuper);
1288 }
else if (ThisOpcode == AMDGPU::V_PK_MOV_B32) {
1329 int64_t &ImmVal)
const {
1330 switch (
MI.getOpcode()) {
1331 case AMDGPU::V_MOV_B32_e32:
1332 case AMDGPU::S_MOV_B32:
1333 case AMDGPU::S_MOVK_I32:
1334 case AMDGPU::S_MOV_B64:
1335 case AMDGPU::V_MOV_B64_e32:
1336 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
1337 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
1338 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
1339 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
1340 case AMDGPU::V_MOV_B64_PSEUDO:
1341 case AMDGPU::V_MOV_B16_t16_e32: {
1344 ImmVal = Src0.getImm();
1345 return MI.getOperand(0).getReg() == Reg;
1350 case AMDGPU::V_MOV_B16_t16_e64: {
1352 if (Src0.isImm() && !
MI.getOperand(1).getImm()) {
1353 ImmVal = Src0.getImm();
1354 return MI.getOperand(0).getReg() == Reg;
1359 case AMDGPU::S_BREV_B32:
1360 case AMDGPU::V_BFREV_B32_e32:
1361 case AMDGPU::V_BFREV_B32_e64: {
1365 return MI.getOperand(0).getReg() == Reg;
1370 case AMDGPU::S_NOT_B32:
1371 case AMDGPU::V_NOT_B32_e32:
1372 case AMDGPU::V_NOT_B32_e64: {
1375 ImmVal =
static_cast<int64_t
>(~static_cast<int32_t>(Src0.getImm()));
1376 return MI.getOperand(0).getReg() == Reg;
1386std::optional<int64_t>
1396 if (!
Op.isReg() || !
Op.getReg().isVirtual())
1397 return std::nullopt;
1399 if (Def && Def->isMoveImmediate()) {
1401 if (ImmSrc.
isImm()) {
1408 return std::nullopt;
1411std::optional<int64_t>
1420 if (RI.isAGPRClass(DstRC))
1421 return AMDGPU::COPY;
1422 if (RI.getRegSizeInBits(*DstRC) == 16) {
1425 return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1427 if (RI.getRegSizeInBits(*DstRC) == 32)
1428 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1429 if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC))
1430 return AMDGPU::S_MOV_B64;
1431 if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC))
1432 return AMDGPU::V_MOV_B64_PSEUDO;
1433 return AMDGPU::COPY;
1438 bool IsIndirectSrc)
const {
1439 if (IsIndirectSrc) {
1441 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1443 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1445 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1447 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1449 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1451 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6);
1453 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7);
1455 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1457 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1459 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1461 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1463 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1465 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1466 if (VecSize <= 1024)
1467 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1473 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1475 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1477 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1479 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1481 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1483 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6);
1485 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7);
1487 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1489 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1491 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1493 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1495 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1497 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1498 if (VecSize <= 1024)
1499 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1506 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1508 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1510 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1512 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1514 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1516 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1518 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1520 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1522 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1524 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1526 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1528 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1530 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1531 if (VecSize <= 1024)
1532 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1539 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1541 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1543 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1545 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1547 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1549 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1551 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1553 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1555 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1557 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1559 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1561 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1563 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1564 if (VecSize <= 1024)
1565 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1572 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1574 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1576 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1578 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1579 if (VecSize <= 1024)
1580 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1587 bool IsSGPR)
const {
1599 assert(EltSize == 32 &&
"invalid reg indexing elt size");
1606 return NeedsCFI ? AMDGPU::SI_SPILL_S32_CFI_SAVE : AMDGPU::SI_SPILL_S32_SAVE;
1608 return NeedsCFI ? AMDGPU::SI_SPILL_S64_CFI_SAVE : AMDGPU::SI_SPILL_S64_SAVE;
1610 return NeedsCFI ? AMDGPU::SI_SPILL_S96_CFI_SAVE : AMDGPU::SI_SPILL_S96_SAVE;
1612 return NeedsCFI ? AMDGPU::SI_SPILL_S128_CFI_SAVE
1613 : AMDGPU::SI_SPILL_S128_SAVE;
1615 return NeedsCFI ? AMDGPU::SI_SPILL_S160_CFI_SAVE
1616 : AMDGPU::SI_SPILL_S160_SAVE;
1618 return NeedsCFI ? AMDGPU::SI_SPILL_S192_CFI_SAVE
1619 : AMDGPU::SI_SPILL_S192_SAVE;
1621 return NeedsCFI ? AMDGPU::SI_SPILL_S224_CFI_SAVE
1622 : AMDGPU::SI_SPILL_S224_SAVE;
1624 return AMDGPU::SI_SPILL_S256_SAVE;
1626 return AMDGPU::SI_SPILL_S288_SAVE;
1628 return AMDGPU::SI_SPILL_S320_SAVE;
1630 return AMDGPU::SI_SPILL_S352_SAVE;
1632 return AMDGPU::SI_SPILL_S384_SAVE;
1634 return NeedsCFI ? AMDGPU::SI_SPILL_S512_CFI_SAVE
1635 : AMDGPU::SI_SPILL_S512_SAVE;
1637 return NeedsCFI ? AMDGPU::SI_SPILL_S1024_CFI_SAVE
1638 : AMDGPU::SI_SPILL_S1024_SAVE;
1647 return AMDGPU::SI_SPILL_V16_SAVE;
1649 return NeedsCFI ? AMDGPU::SI_SPILL_V32_CFI_SAVE : AMDGPU::SI_SPILL_V32_SAVE;
1651 return NeedsCFI ? AMDGPU::SI_SPILL_V64_CFI_SAVE : AMDGPU::SI_SPILL_V64_SAVE;
1653 return NeedsCFI ? AMDGPU::SI_SPILL_V96_CFI_SAVE : AMDGPU::SI_SPILL_V96_SAVE;
1655 return NeedsCFI ? AMDGPU::SI_SPILL_V128_CFI_SAVE
1656 : AMDGPU::SI_SPILL_V128_SAVE;
1658 return NeedsCFI ? AMDGPU::SI_SPILL_V160_CFI_SAVE
1659 : AMDGPU::SI_SPILL_V160_SAVE;
1661 return NeedsCFI ? AMDGPU::SI_SPILL_V192_CFI_SAVE
1662 : AMDGPU::SI_SPILL_V192_SAVE;
1664 return NeedsCFI ? AMDGPU::SI_SPILL_V224_CFI_SAVE
1665 : AMDGPU::SI_SPILL_V224_SAVE;
1667 return NeedsCFI ? AMDGPU::SI_SPILL_V256_CFI_SAVE
1668 : AMDGPU::SI_SPILL_V256_SAVE;
1670 return NeedsCFI ? AMDGPU::SI_SPILL_V288_CFI_SAVE
1671 : AMDGPU::SI_SPILL_V288_SAVE;
1673 return NeedsCFI ? AMDGPU::SI_SPILL_V320_CFI_SAVE
1674 : AMDGPU::SI_SPILL_V320_SAVE;
1676 return NeedsCFI ? AMDGPU::SI_SPILL_V352_CFI_SAVE
1677 : AMDGPU::SI_SPILL_V352_SAVE;
1679 return NeedsCFI ? AMDGPU::SI_SPILL_V384_CFI_SAVE
1680 : AMDGPU::SI_SPILL_V384_SAVE;
1682 return NeedsCFI ? AMDGPU::SI_SPILL_V512_CFI_SAVE
1683 : AMDGPU::SI_SPILL_V512_SAVE;
1685 return NeedsCFI ? AMDGPU::SI_SPILL_V1024_CFI_SAVE
1686 : AMDGPU::SI_SPILL_V1024_SAVE;
1695 return NeedsCFI ? AMDGPU::SI_SPILL_AV32_CFI_SAVE
1696 : AMDGPU::SI_SPILL_AV32_SAVE;
1698 return NeedsCFI ? AMDGPU::SI_SPILL_AV64_CFI_SAVE
1699 : AMDGPU::SI_SPILL_AV64_SAVE;
1701 return NeedsCFI ? AMDGPU::SI_SPILL_AV96_CFI_SAVE
1702 : AMDGPU::SI_SPILL_AV96_SAVE;
1704 return NeedsCFI ? AMDGPU::SI_SPILL_AV128_CFI_SAVE
1705 : AMDGPU::SI_SPILL_AV128_SAVE;
1707 return NeedsCFI ? AMDGPU::SI_SPILL_AV160_CFI_SAVE
1708 : AMDGPU::SI_SPILL_AV160_SAVE;
1710 return NeedsCFI ? AMDGPU::SI_SPILL_AV192_CFI_SAVE
1711 : AMDGPU::SI_SPILL_AV192_SAVE;
1713 return NeedsCFI ? AMDGPU::SI_SPILL_AV224_CFI_SAVE
1714 : AMDGPU::SI_SPILL_AV224_SAVE;
1716 return NeedsCFI ? AMDGPU::SI_SPILL_AV256_CFI_SAVE
1717 : AMDGPU::SI_SPILL_AV256_SAVE;
1719 return AMDGPU::SI_SPILL_AV288_SAVE;
1721 return AMDGPU::SI_SPILL_AV320_SAVE;
1723 return AMDGPU::SI_SPILL_AV352_SAVE;
1725 return AMDGPU::SI_SPILL_AV384_SAVE;
1727 return NeedsCFI ? AMDGPU::SI_SPILL_AV512_CFI_SAVE
1728 : AMDGPU::SI_SPILL_AV512_SAVE;
1730 return NeedsCFI ? AMDGPU::SI_SPILL_AV1024_CFI_SAVE
1731 : AMDGPU::SI_SPILL_AV1024_SAVE;
1738 bool IsVectorSuperClass) {
1743 if (IsVectorSuperClass)
1744 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1746 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1752 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1759 if (ST.hasMAIInsts())
1765void SIInstrInfo::storeRegToStackSlotImpl(
1778 FrameInfo.getObjectAlign(FrameIndex));
1779 unsigned SpillSize = RI.getSpillSize(*RC);
1785 assert(SrcReg != AMDGPU::M0 &&
"m0 should not be spilled");
1786 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1787 SrcReg != AMDGPU::EXEC &&
"exec should not be spilled");
1796 if (SrcReg.
isVirtual() && SpillSize == 4) {
1797 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1810 SpillSize, *MFI, NeedsCFI);
1825 storeRegToStackSlotImpl(
MBB,
MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
1834 storeRegToStackSlotImpl(
MBB,
MI, SrcReg, isKill, FrameIndex, RC,
Register(),
1841 return AMDGPU::SI_SPILL_S32_RESTORE;
1843 return AMDGPU::SI_SPILL_S64_RESTORE;
1845 return AMDGPU::SI_SPILL_S96_RESTORE;
1847 return AMDGPU::SI_SPILL_S128_RESTORE;
1849 return AMDGPU::SI_SPILL_S160_RESTORE;
1851 return AMDGPU::SI_SPILL_S192_RESTORE;
1853 return AMDGPU::SI_SPILL_S224_RESTORE;
1855 return AMDGPU::SI_SPILL_S256_RESTORE;
1857 return AMDGPU::SI_SPILL_S288_RESTORE;
1859 return AMDGPU::SI_SPILL_S320_RESTORE;
1861 return AMDGPU::SI_SPILL_S352_RESTORE;
1863 return AMDGPU::SI_SPILL_S384_RESTORE;
1865 return AMDGPU::SI_SPILL_S512_RESTORE;
1867 return AMDGPU::SI_SPILL_S1024_RESTORE;
1876 return AMDGPU::SI_SPILL_V16_RESTORE;
1878 return AMDGPU::SI_SPILL_V32_RESTORE;
1880 return AMDGPU::SI_SPILL_V64_RESTORE;
1882 return AMDGPU::SI_SPILL_V96_RESTORE;
1884 return AMDGPU::SI_SPILL_V128_RESTORE;
1886 return AMDGPU::SI_SPILL_V160_RESTORE;
1888 return AMDGPU::SI_SPILL_V192_RESTORE;
1890 return AMDGPU::SI_SPILL_V224_RESTORE;
1892 return AMDGPU::SI_SPILL_V256_RESTORE;
1894 return AMDGPU::SI_SPILL_V288_RESTORE;
1896 return AMDGPU::SI_SPILL_V320_RESTORE;
1898 return AMDGPU::SI_SPILL_V352_RESTORE;
1900 return AMDGPU::SI_SPILL_V384_RESTORE;
1902 return AMDGPU::SI_SPILL_V512_RESTORE;
1904 return AMDGPU::SI_SPILL_V1024_RESTORE;
1913 return AMDGPU::SI_SPILL_AV32_RESTORE;
1915 return AMDGPU::SI_SPILL_AV64_RESTORE;
1917 return AMDGPU::SI_SPILL_AV96_RESTORE;
1919 return AMDGPU::SI_SPILL_AV128_RESTORE;
1921 return AMDGPU::SI_SPILL_AV160_RESTORE;
1923 return AMDGPU::SI_SPILL_AV192_RESTORE;
1925 return AMDGPU::SI_SPILL_AV224_RESTORE;
1927 return AMDGPU::SI_SPILL_AV256_RESTORE;
1929 return AMDGPU::SI_SPILL_AV288_RESTORE;
1931 return AMDGPU::SI_SPILL_AV320_RESTORE;
1933 return AMDGPU::SI_SPILL_AV352_RESTORE;
1935 return AMDGPU::SI_SPILL_AV384_RESTORE;
1937 return AMDGPU::SI_SPILL_AV512_RESTORE;
1939 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1946 bool IsVectorSuperClass) {
1951 if (IsVectorSuperClass)
1952 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1954 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1960 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1967 if (ST.hasMAIInsts())
1970 assert(!RI.isAGPRClass(RC));
1984 unsigned SpillSize = RI.getSpillSize(*RC);
1991 FrameInfo.getObjectAlign(FrameIndex));
1993 if (RI.isSGPRClass(RC)) {
1996 assert(DestReg != AMDGPU::M0 &&
"m0 should not be reloaded into");
1997 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1998 DestReg != AMDGPU::EXEC &&
"exec should not be spilled");
2003 if (DestReg.
isVirtual() && SpillSize == 4) {
2005 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
2032 unsigned Quantity)
const {
2034 unsigned MaxSNopCount = 1u << ST.getSNopBits();
2035 while (Quantity > 0) {
2036 unsigned Arg = std::min(Quantity, MaxSNopCount);
2047 constexpr unsigned DoorbellIDMask = 0x3ff;
2048 constexpr unsigned ECQueueWaveAbort = 0x400;
2053 if (!
MBB.succ_empty() ||
2055 false) !=
MBB.end()) {
2056 MBB.splitAt(
MI,
false);
2060 MBB.addSuccessor(TrapBB);
2066 Register DoorbellReg =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2070 BuildMI(*TrapBB, TrapBB->
end(),
DL,
get(AMDGPU::S_MOV_B32), AMDGPU::TTMP2)
2073 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2074 BuildMI(*TrapBB, TrapBB->
end(),
DL,
get(AMDGPU::S_AND_B32), DoorbellRegMasked)
2079 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2080 BuildMI(*TrapBB, TrapBB->
end(),
DL,
get(AMDGPU::S_OR_B32), SetWaveAbortBit)
2081 .
addUse(DoorbellRegMasked)
2082 .
addImm(ECQueueWaveAbort)
2084 BuildMI(*TrapBB, TrapBB->
end(),
DL,
get(AMDGPU::S_MOV_B32), AMDGPU::M0)
2085 .
addUse(SetWaveAbortBit);
2088 BuildMI(*TrapBB, TrapBB->
end(),
DL,
get(AMDGPU::S_MOV_B32), AMDGPU::M0)
2099 return MBB.getNextNode();
2103 switch (
MI.getOpcode()) {
2105 if (
MI.isMetaInstruction())
2110 return MI.getOperand(0).getImm() + 1;
2123 const bool IsStore = LdSt.mayStore();
2126 unsigned Offset = LdSt.getOffsetOp().getImm();
2127 unsigned NumDwords = LdSt.getBitWidth() / 32;
2129 const bool UseGPRIdxMode = LdSt.isGPRIdx();
2130 assert(UseGPRIdxMode ==
TII.getSubtarget().useVGPRIndexMode());
2133 if (UseGPRIdxMode) {
2135 .
add(LdSt.getIdxOp())
2141 if (UseGPRIdxMode) {
2142 Opcode = IsStore ? AMDGPU::V_MOV_B32_indirect_write
2143 : AMDGPU::V_MOV_B32_indirect_read;
2146 IsStore ? AMDGPU::V_MOVRELD_B32_as_mem : AMDGPU::V_MOVRELS_B32_as_mem;
2154 for (
unsigned i = 0; i < NumDwords; ++i) {
2176 if (!UseGPRIdxMode) {
2190 MI.eraseFromParent();
2203 switch (
MI.getOpcode()) {
2205 case AMDGPU::S_MOV_B64_term:
2208 MI.setDesc(
get(AMDGPU::S_MOV_B64));
2211 case AMDGPU::S_MOV_B32_term:
2214 MI.setDesc(
get(AMDGPU::S_MOV_B32));
2217 case AMDGPU::S_XOR_B64_term:
2220 MI.setDesc(
get(AMDGPU::S_XOR_B64));
2223 case AMDGPU::S_XOR_B32_term:
2226 MI.setDesc(
get(AMDGPU::S_XOR_B32));
2228 case AMDGPU::S_OR_B64_term:
2231 MI.setDesc(
get(AMDGPU::S_OR_B64));
2233 case AMDGPU::S_OR_B32_term:
2236 MI.setDesc(
get(AMDGPU::S_OR_B32));
2239 case AMDGPU::S_ANDN2_B64_term:
2242 MI.setDesc(
get(AMDGPU::S_ANDN2_B64));
2245 case AMDGPU::S_ANDN2_B32_term:
2248 MI.setDesc(
get(AMDGPU::S_ANDN2_B32));
2251 case AMDGPU::S_AND_B64_term:
2254 MI.setDesc(
get(AMDGPU::S_AND_B64));
2257 case AMDGPU::S_AND_B32_term:
2260 MI.setDesc(
get(AMDGPU::S_AND_B32));
2263 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2266 MI.setDesc(
get(AMDGPU::S_AND_SAVEEXEC_B64));
2269 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2272 MI.setDesc(
get(AMDGPU::S_AND_SAVEEXEC_B32));
2275 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
2276 MI.setDesc(
get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
2278 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
2279 MI.setDesc(
get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
2282 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2283 MI.setDesc(
get(AMDGPU::V_WRITELANE_B32));
2286 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2287 MI.setDesc(
get(AMDGPU::V_READLANE_B32));
2289 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
2293 get(IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
2296 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
2299 int64_t
Imm =
MI.getOperand(1).getImm();
2301 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2302 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2307 MI.eraseFromParent();
2313 case AMDGPU::V_MOV_B64_PSEUDO: {
2315 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2316 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2324 if (ST.hasVMovB64Inst() && Mov64RC->
contains(Dst)) {
2325 MI.setDesc(Mov64Desc);
2329 (
SrcOp.isGlobal() && ST.has64BitLiterals()))
2332 if (
SrcOp.isGlobal()) {
2343 }
else if (
SrcOp.isImm()) {
2345 APInt Lo(32,
Imm.getLoBits(32).getZExtValue());
2346 APInt Hi(32,
Imm.getHiBits(32).getZExtValue());
2370 if (ST.hasPkMovB32() &&
2389 MI.eraseFromParent();
2392 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2396 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2400 if (ST.has64BitLiterals()) {
2401 MI.setDesc(
get(AMDGPU::S_MOV_B64));
2405 if (
SrcOp.isGlobal()) {
2407 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2408 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2417 MI.eraseFromParent();
2424 MI.setDesc(
get(AMDGPU::S_MOV_B64));
2429 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2430 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2432 APInt Lo(32,
Imm.getLoBits(32).getZExtValue());
2433 APInt Hi(32,
Imm.getHiBits(32).getZExtValue());
2438 MI.eraseFromParent();
2441 case AMDGPU::V_SET_INACTIVE_B32: {
2445 .
add(
MI.getOperand(3))
2446 .
add(
MI.getOperand(4))
2447 .
add(
MI.getOperand(1))
2448 .
add(
MI.getOperand(2))
2449 .
add(
MI.getOperand(5));
2450 MI.eraseFromParent();
2453 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2454 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2455 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2456 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2457 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2458 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2459 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2460 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2461 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2462 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2463 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2464 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2465 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2466 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2467 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2468 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2469 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2470 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2471 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2472 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2473 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2474 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2475 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2476 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2477 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2478 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2479 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2480 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2481 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2482 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2483 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2484 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2485 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2489 if (RI.hasVGPRs(EltRC)) {
2490 Opc = AMDGPU::V_MOVRELD_B32_e32;
2492 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2493 : AMDGPU::S_MOVRELD_B32;
2498 bool IsUndef =
MI.getOperand(1).isUndef();
2499 unsigned SubReg =
MI.getOperand(3).getImm();
2500 assert(VecReg ==
MI.getOperand(1).getReg());
2505 .
add(
MI.getOperand(2))
2509 const int ImpDefIdx =
2511 const int ImpUseIdx = ImpDefIdx + 1;
2513 MI.eraseFromParent();
2516 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2517 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2518 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2519 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2520 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2521 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6:
2522 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7:
2523 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2524 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2525 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2526 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2527 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2528 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2529 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2530 assert(ST.useVGPRIndexMode());
2532 bool IsUndef =
MI.getOperand(1).isUndef();
2541 const MCInstrDesc &OpDesc =
get(AMDGPU::V_MOV_B32_indirect_write);
2545 .
add(
MI.getOperand(2))
2549 const int ImpDefIdx =
2551 const int ImpUseIdx = ImpDefIdx + 1;
2558 MI.eraseFromParent();
2561 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2562 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2563 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2564 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2565 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2566 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6:
2567 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7:
2568 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2569 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2570 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2571 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2572 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2573 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2574 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2575 assert(ST.useVGPRIndexMode());
2578 bool IsUndef =
MI.getOperand(1).isUndef();
2582 .
add(
MI.getOperand(2))
2595 MI.eraseFromParent();
2598 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2601 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
2602 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
2621 if (ST.hasGetPCZeroExtension()) {
2625 BuildMI(MF,
DL,
get(AMDGPU::S_SEXT_I32_I16), RegHi).addReg(RegHi));
2632 BuildMI(MF,
DL,
get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo));
2642 MI.eraseFromParent();
2645 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {
2655 Op.setOffset(
Op.getOffset() + 4);
2657 BuildMI(MF,
DL,
get(AMDGPU::S_ADD_U64), Reg).addReg(Reg).add(
Op));
2661 MI.eraseFromParent();
2664 case AMDGPU::ENTER_STRICT_WWM: {
2670 case AMDGPU::ENTER_STRICT_WQM: {
2677 MI.eraseFromParent();
2680 case AMDGPU::EXIT_STRICT_WWM:
2681 case AMDGPU::EXIT_STRICT_WQM: {
2687 case AMDGPU::SI_RETURN: {
2701 MI.eraseFromParent();
2705 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2706 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2707 MI.setDesc(
get(AMDGPU::S_MUL_U64));
2710 case AMDGPU::S_GETPC_B64_pseudo:
2711 MI.setDesc(
get(AMDGPU::S_GETPC_B64));
2712 if (ST.hasGetPCZeroExtension()) {
2714 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2723 case AMDGPU::V_MAX_BF16_PSEUDO_e64: {
2724 assert(ST.hasBF16PackedInsts());
2725 MI.setDesc(
get(AMDGPU::V_PK_MAX_NUM_BF16));
2736 case AMDGPU::GET_STACK_BASE:
2739 if (ST.getFrameLowering()->mayReserveScratchForCWSR(*
MBB.getParent())) {
2746 Register DestReg =
MI.getOperand(0).getReg();
2756 MI.getOperand(
MI.getNumExplicitOperands()).setIsDead(
false);
2757 MI.getOperand(
MI.getNumExplicitOperands()).setIsUse();
2758 MI.setDesc(
get(AMDGPU::S_CMOVK_I32));
2761 MI.setDesc(
get(AMDGPU::S_MOV_B32));
2764 MI.getNumExplicitOperands());
2782 case AMDGPU::S_MOV_B64:
2783 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2792 if (UsedLanes.
all())
2797 unsigned LoSubReg = RI.composeSubRegIndices(OrigSubReg, AMDGPU::sub0);
2798 unsigned HiSubReg = RI.composeSubRegIndices(OrigSubReg, AMDGPU::sub1);
2800 bool NeedLo = (UsedLanes & RI.getSubRegIndexLaneMask(LoSubReg)).any();
2801 bool NeedHi = (UsedLanes & RI.getSubRegIndexLaneMask(HiSubReg)).any();
2803 if (NeedLo && NeedHi)
2807 int32_t Imm32 = NeedLo ?
Lo_32(Imm64) :
Hi_32(Imm64);
2809 unsigned UseSubReg = NeedLo ? LoSubReg : HiSubReg;
2818 case AMDGPU::S_LOAD_DWORDX16_IMM:
2819 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2820 if (SubIdx != 0 || UsedLanes.
all())
2831 assert(
MRI.use_nodbg_empty(DestReg) &&
"DestReg should have no users yet.");
2838 UseMO !=
I->findRegisterUseOperand(RegToFind,
nullptr) ||
2839 UseMO->
getSubReg() == AMDGPU::NoSubRegister)
2843 unsigned SubregSize = RI.getSubRegIdxSize(UseMO->
getSubReg());
2845 unsigned NewOpcode = -1;
2846 if (SubregSize == 256)
2847 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2848 else if (SubregSize == 128)
2849 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2856 MRI.setRegClass(DestReg, NewRC);
2859 UseMO->
setSubReg(AMDGPU::NoSubRegister);
2864 MI->getOperand(0).setReg(DestReg);
2865 MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister);
2869 OffsetMO->
setImm(FinalOffset);
2875 MI->setMemRefs(*MF, NewMMOs);
2888std::pair<MachineInstr*, MachineInstr*>
2890 assert (
MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2892 if (ST.hasVMovB64Inst() && ST.hasFeature(AMDGPU::FeatureDPALU_DPP) &&
2895 MI.setDesc(
get(AMDGPU::V_MOV_B64_dpp));
2896 return std::pair(&
MI,
nullptr);
2907 for (
auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2909 if (Dst.isPhysical()) {
2910 MovDPP.addDef(RI.getSubReg(Dst,
Sub));
2913 auto Tmp =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2917 for (
unsigned I = 1;
I <= 2; ++
I) {
2920 if (
SrcOp.isImm()) {
2922 Imm.ashrInPlace(Part * 32);
2923 MovDPP.addImm(
Imm.getLoBits(32).getZExtValue());
2927 if (Src.isPhysical())
2928 MovDPP.addReg(RI.getSubReg(Src,
Sub));
2935 MovDPP.addImm(MO.getImm());
2937 Split[Part] = MovDPP;
2941 if (Dst.isVirtual())
2948 MI.eraseFromParent();
2949 return std::pair(Split[0], Split[1]);
2952std::optional<DestSourcePair>
2954 if (
MI.getOpcode() == AMDGPU::WWM_COPY)
2957 return std::nullopt;
2961 AMDGPU::OpName Src0OpName,
2963 AMDGPU::OpName Src1OpName)
const {
2970 "All commutable instructions have both src0 and src1 modifiers");
2972 int Src0ModsVal = Src0Mods->
getImm();
2973 int Src1ModsVal = Src1Mods->
getImm();
2975 Src1Mods->
setImm(Src0ModsVal);
2976 Src0Mods->
setImm(Src1ModsVal);
2985 bool IsKill = RegOp.
isKill();
2987 bool IsUndef = RegOp.
isUndef();
2988 bool IsDebug = RegOp.
isDebug();
2990 if (NonRegOp.
isImm())
2992 else if (NonRegOp.
isFI())
3013 int64_t NonRegVal = NonRegOp1.
getImm();
3016 NonRegOp2.
setImm(NonRegVal);
3026 bool CheckSwappedSrc =
false) {
3027 unsigned Opc =
MI.getOpcode();
3028 if (!
TII.getSubtarget().hasPkF32InvalidOpSel() ||
3029 (
Opc != AMDGPU::V_PK_ADD_F32 &&
Opc != AMDGPU::V_PK_MUL_F32 &&
3030 Opc != AMDGPU::V_PK_FMA_F32))
3033 auto Mods = [&](AMDGPU::OpName Name) {
3035 return Op ?
static_cast<unsigned>(
Op->getImm()) : 0u;
3037 unsigned Src0Mods = Mods(AMDGPU::OpName::src0_modifiers);
3038 unsigned Src1Mods = Mods(AMDGPU::OpName::src1_modifiers);
3039 unsigned Src2Mods = Mods(AMDGPU::OpName::src2_modifiers);
3041 if (CheckSwappedSrc)
3044 if (
Opc == AMDGPU::V_PK_FMA_F32)
3050 unsigned OpIdx1)
const {
3055 unsigned Opc =
MI.getOpcode();
3056 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
3057 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
3061 bool IsSrc01Swap = ((int)OpIdx0 == Src0Idx && (int)OpIdx1 == Src1Idx) ||
3062 ((
int)OpIdx0 == Src1Idx && (
int)OpIdx1 == Src0Idx);
3074 if ((
int)OpIdx0 == Src0Idx && !MO0.
isReg() &&
3077 if ((
int)OpIdx1 == Src0Idx && !MO1.
isReg() &&
3082 if ((
int)OpIdx1 != Src0Idx && MO0.
isReg()) {
3088 if ((
int)OpIdx0 != Src0Idx && MO1.
isReg()) {
3110 unsigned Src1Idx)
const {
3111 assert(!NewMI &&
"this should never be used");
3116 unsigned Opc =
MI.getOpcode();
3118 if (CommutedOpcode == -1)
3121 if (Src0Idx > Src1Idx)
3124 assert(AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0) ==
3125 static_cast<int>(Src0Idx) &&
3126 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1) ==
3127 static_cast<int>(Src1Idx) &&
3128 "inconsistency with findCommutedOpIndices");
3136 if (Src0.isReg() && Src1.isReg()) {
3140 }
else if (Src0.isReg() && !Src1.isReg()) {
3142 }
else if (!Src0.isReg() && Src1.isReg()) {
3144 }
else if (Src0.isImm() && Src1.isImm()) {
3153 Src1, AMDGPU::OpName::src1_modifiers);
3156 AMDGPU::OpName::src1_sel);
3168 unsigned &SrcOpIdx0,
3169 unsigned &SrcOpIdx1)
const {
3177 unsigned &SrcOpIdx0,
3178 unsigned &SrcOpIdx1)
const {
3179 if (!
Desc.isCommutable())
3182 unsigned Opc =
Desc.getOpcode();
3183 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
3187 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
3191 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
3195 int64_t BrOffset)
const {
3207 return isIntN(ST.getCLOpts().s_branch_bits, BrOffset);
3212 return MI.getOperand(0).getMBB();
3217 if (
MI.getOpcode() == AMDGPU::SI_IF ||
MI.getOpcode() == AMDGPU::SI_ELSE ||
3218 MI.getOpcode() == AMDGPU::SI_LOOP ||
3219 MI.getOpcode() == AMDGPU::SI_WATERFALL_LOOP)
3231 "new block should be inserted for expanding unconditional branch");
3234 "restore block should be inserted for restoring clobbered registers");
3242 if (ST.useAddPCI64Inst()) {
3244 MCCtx.createTempSymbol(
"offset",
true);
3248 MCCtx.createTempSymbol(
"post_addpc",
true);
3249 AddPC->setPostInstrSymbol(*MF, PostAddPCLabel);
3253 Offset->setVariableValue(OffsetExpr);
3257 assert(RS &&
"RegScavenger required for long branching");
3261 Register PCReg =
MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3265 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
3266 ST.hasVALUReadSGPRHazard();
3267 auto ApplyHazardWorkarounds = [
this, &
MBB, &
I, &
DL, FlushSGPRWrites]() {
3268 if (FlushSGPRWrites)
3276 ApplyHazardWorkarounds();
3279 MCCtx.createTempSymbol(
"post_getpc",
true);
3283 MCCtx.createTempSymbol(
"offset_lo",
true);
3285 MCCtx.createTempSymbol(
"offset_hi",
true);
3288 .
addReg(PCReg, {}, AMDGPU::sub0)
3292 .
addReg(PCReg, {}, AMDGPU::sub1)
3294 ApplyHazardWorkarounds();
3335 if (LongBranchReservedReg) {
3336 RS->enterBasicBlock(
MBB);
3337 Scav = LongBranchReservedReg;
3339 RS->enterBasicBlockEnd(
MBB);
3340 Scav = RS->scavengeRegisterBackwards(
3345 RS->setRegUsed(Scav);
3346 MRI.replaceRegWith(PCReg, Scav);
3347 MRI.clearVirtRegs();
3353 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS);
3354 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1);
3355 MRI.clearVirtRegs();
3370unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate
Cond) {
3372 case SIInstrInfo::SCC_TRUE:
3373 return AMDGPU::S_CBRANCH_SCC1;
3374 case SIInstrInfo::SCC_FALSE:
3375 return AMDGPU::S_CBRANCH_SCC0;
3376 case SIInstrInfo::VCCNZ:
3377 return AMDGPU::S_CBRANCH_VCCNZ;
3378 case SIInstrInfo::VCCZ:
3379 return AMDGPU::S_CBRANCH_VCCZ;
3380 case SIInstrInfo::EXECNZ:
3381 return AMDGPU::S_CBRANCH_EXECNZ;
3382 case SIInstrInfo::EXECZ:
3383 return AMDGPU::S_CBRANCH_EXECZ;
3389SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(
unsigned Opcode) {
3391 case AMDGPU::S_CBRANCH_SCC0:
3393 case AMDGPU::S_CBRANCH_SCC1:
3395 case AMDGPU::S_CBRANCH_VCCNZ:
3397 case AMDGPU::S_CBRANCH_VCCZ:
3399 case AMDGPU::S_CBRANCH_EXECNZ:
3401 case AMDGPU::S_CBRANCH_EXECZ:
3413 bool AllowModify)
const {
3414 if (
I->getOpcode() == AMDGPU::S_BRANCH) {
3416 TBB =
I->getOperand(0).getMBB();
3420 BranchPredicate Pred = getBranchPredicate(
I->getOpcode());
3421 if (Pred == INVALID_BR)
3426 Cond.push_back(
I->getOperand(1));
3430 if (
I ==
MBB.end()) {
3436 if (
I->getOpcode() == AMDGPU::S_BRANCH) {
3438 FBB =
I->getOperand(0).getMBB();
3448 bool AllowModify)
const {
3456 while (
I != E && !
I->isBranch() && !
I->isReturn()) {
3457 switch (
I->getOpcode()) {
3458 case AMDGPU::S_MOV_B64_term:
3459 case AMDGPU::S_XOR_B64_term:
3460 case AMDGPU::S_OR_B64_term:
3461 case AMDGPU::S_ANDN2_B64_term:
3462 case AMDGPU::S_AND_B64_term:
3463 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3464 case AMDGPU::S_MOV_B32_term:
3465 case AMDGPU::S_XOR_B32_term:
3466 case AMDGPU::S_OR_B32_term:
3467 case AMDGPU::S_ANDN2_B32_term:
3468 case AMDGPU::S_AND_B32_term:
3469 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3470 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
3471 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
3474 case AMDGPU::SI_ELSE:
3475 case AMDGPU::SI_KILL_I1_TERMINATOR:
3476 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3493 int *BytesRemoved)
const {
3495 unsigned RemovedSize = 0;
3498 if (
MI.isBranch() ||
MI.isReturn()) {
3500 MI.eraseFromParent();
3506 *BytesRemoved = RemovedSize;
3523 int *BytesAdded)
const {
3524 if (!FBB &&
Cond.empty()) {
3528 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3535 = getBranchOpcode(
static_cast<BranchPredicate
>(
Cond[0].
getImm()));
3547 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3565 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3572 if (
Cond.size() != 2) {
3576 if (
Cond[0].isImm()) {
3597 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
3601 std::optional<bool> createTripCountGreaterCondition(
3602 int TC, MachineBasicBlock &
MBB,
3603 SmallVectorImpl<MachineOperand> &CondParam)
override {
3604 CondParam = this->
Cond;
3608 void adjustTripCount(
int TripCountAdjust)
override {}
3610 void setPreheader(MachineBasicBlock *NewPreheader)
override {}
3614std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
3623 if (
TBB == LoopBB && FBB == LoopBB)
3630 assert((
TBB == LoopBB || FBB == LoopBB) &&
3631 "The Loop must be a single-basic-block loop");
3634 BranchPredicate Pred =
static_cast<BranchPredicate
>(
Cond[0].getImm());
3635 if (Pred != SCC_TRUE && Pred != SCC_FALSE)
3640 if (
MI.isCall() ||
MI.isInlineAsm())
3655 if (CmpI == Instructions.end() || CmpI->isPHI())
3659 return std::make_unique<AMDGPUPipelinerLoopInfo>(
CmpInst,
Cond);
3666 SelectExpansion &Exp)
const {
3672 if (
MRI.getRegClass(FalseReg) != RC)
3676 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts;
3679 return RI.hasVGPRs(RC) && NumInsts <= 6;
3687 if (
MRI.getRegClass(FalseReg) != RC)
3693 if (NumInsts % 2 == 0)
3696 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts;
3697 return RI.isSGPRClass(RC);
3708 BranchPredicate Pred =
static_cast<BranchPredicate
>(
Cond[0].getImm());
3709 if (Pred == VCCZ || Pred == SCC_FALSE) {
3710 Pred =
static_cast<BranchPredicate
>(-Pred);
3716 unsigned DstSize = RI.getRegSizeInBits(*DstRC);
3718 if (DstSize == 32) {
3720 if (Pred == SCC_TRUE) {
3735 if (DstSize == 64 && Pred == SCC_TRUE) {
3745 static const int16_t Sub0_15[] = {
3746 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3747 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3748 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3749 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3752 static const int16_t Sub0_15_64[] = {
3753 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3754 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3755 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3756 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3759 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3761 const int16_t *SubIndices = Sub0_15;
3762 int NElts = DstSize / 32;
3766 if (Pred == SCC_TRUE) {
3768 SelOp = AMDGPU::S_CSELECT_B32;
3769 EltRC = &AMDGPU::SGPR_32RegClass;
3771 SelOp = AMDGPU::S_CSELECT_B64;
3772 EltRC = &AMDGPU::SGPR_64RegClass;
3773 SubIndices = Sub0_15_64;
3779 MBB,
I,
DL,
get(AMDGPU::REG_SEQUENCE), DstReg);
3784 for (
int Idx = 0; Idx != NElts; ++Idx) {
3785 Register DstElt =
MRI.createVirtualRegister(EltRC);
3788 unsigned SubIdx = SubIndices[Idx];
3791 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3793 .
addReg(FalseReg, {}, SubIdx)
3794 .addReg(TrueReg, {}, SubIdx);
3797 .
addReg(TrueReg, {}, SubIdx)
3798 .addReg(FalseReg, {}, SubIdx);
3811 if (
MI.isBranch() ||
MI.isCall() ||
MI.isReturn() ||
MI.isIndirectBranch())
3814 switch (
MI.getOpcode()) {
3815 case AMDGPU::S_ENDPGM:
3816 case AMDGPU::S_ENDPGM_SAVED:
3817 case AMDGPU::S_TRAP:
3818 case AMDGPU::S_GETREG_B32:
3819 case AMDGPU::S_SETREG_B32:
3820 case AMDGPU::S_SETREG_B32_mode:
3821 case AMDGPU::S_SETREG_IMM32_B32:
3822 case AMDGPU::S_SETREG_IMM32_B32_mode:
3823 case AMDGPU::S_SENDMSG:
3824 case AMDGPU::S_SENDMSGHALT:
3825 case AMDGPU::S_SENDMSG_RTN_B32:
3826 case AMDGPU::S_SENDMSG_RTN_B64:
3827 case AMDGPU::S_BARRIER_WAIT:
3828 case AMDGPU::S_BARRIER_SIGNAL_M0:
3829 case AMDGPU::S_BARRIER_SIGNAL_IMM:
3830 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
3831 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
3839 switch (
MI.getOpcode()) {
3840 case AMDGPU::V_MOV_B16_t16_e32:
3841 case AMDGPU::V_MOV_B16_t16_e64:
3842 case AMDGPU::V_MOV_B32_e32:
3843 case AMDGPU::V_MOV_B32_e64:
3844 case AMDGPU::V_MOV_B64_PSEUDO:
3845 case AMDGPU::V_MOV_B64_e32:
3846 case AMDGPU::V_MOV_B64_e64:
3847 case AMDGPU::S_MOV_B32:
3848 case AMDGPU::S_MOV_B64:
3849 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3851 case AMDGPU::WWM_COPY:
3852 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3853 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3854 case AMDGPU::V_ACCVGPR_MOV_B32:
3855 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3856 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3864 switch (
MI.getOpcode()) {
3865 case AMDGPU::V_MOV_B16_t16_e32:
3866 case AMDGPU::V_MOV_B16_t16_e64:
3868 case AMDGPU::V_MOV_B32_e32:
3869 case AMDGPU::V_MOV_B32_e64:
3870 case AMDGPU::V_MOV_B64_PSEUDO:
3871 case AMDGPU::V_MOV_B64_e32:
3872 case AMDGPU::V_MOV_B64_e64:
3873 case AMDGPU::S_MOV_B32:
3874 case AMDGPU::S_MOV_B64:
3875 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3877 case AMDGPU::WWM_COPY:
3878 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3879 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3880 case AMDGPU::V_ACCVGPR_MOV_B32:
3881 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3882 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3890 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3891 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3892 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3895 unsigned Opc =
MI.getOpcode();
3897 int Idx = AMDGPU::getNamedOperandIdx(
Opc, Name);
3899 MI.removeOperand(Idx);
3905 MI.setDesc(NewDesc);
3911 unsigned NumOps =
Desc.getNumOperands() +
Desc.implicit_uses().size() +
3912 Desc.implicit_defs().size();
3914 for (
unsigned I =
MI.getNumOperands() - 1;
I >=
NumOps; --
I)
3915 MI.removeOperand(
I);
3919 unsigned SubRegIndex) {
3920 switch (SubRegIndex) {
3921 case AMDGPU::NoSubRegister:
3931 case AMDGPU::sub1_lo16:
3933 case AMDGPU::sub1_hi16:
3936 return std::nullopt;
3942std::optional<unsigned>
3944 unsigned SubRegIndex) {
3945 if (SubRegIndex == AMDGPU::NoSubRegister)
3948 unsigned LoReloc, HiReloc;
3949 switch (TargetFlags &
MO_MASK) {
3968 return std::nullopt;
3971 const unsigned BaseFlags = TargetFlags &
~MO_MASK;
3972 switch (SubRegIndex) {
3974 return BaseFlags | LoReloc;
3976 return BaseFlags | HiReloc;
3978 return std::nullopt;
3984 case AMDGPU::V_MAC_F16_e32:
3985 case AMDGPU::V_MAC_F16_e64:
3986 case AMDGPU::V_MAD_F16_e64:
3987 return AMDGPU::V_MADAK_F16;
3988 case AMDGPU::V_MAC_F32_e32:
3989 case AMDGPU::V_MAC_F32_e64:
3990 case AMDGPU::V_MAD_F32_e64:
3991 return AMDGPU::V_MADAK_F32;
3992 case AMDGPU::V_FMAC_F32_e32:
3993 case AMDGPU::V_FMAC_F32_e64:
3994 case AMDGPU::V_FMA_F32_e64:
3995 return AMDGPU::V_FMAAK_F32;
3996 case AMDGPU::V_FMAC_F16_e32:
3997 case AMDGPU::V_FMAC_F16_e64:
3998 case AMDGPU::V_FMAC_F16_t16_e64:
3999 case AMDGPU::V_FMAC_F16_fake16_e64:
4000 case AMDGPU::V_FMAC_F16_t16_e32:
4001 case AMDGPU::V_FMAC_F16_fake16_e32:
4002 case AMDGPU::V_FMA_F16_e64:
4003 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4004 ? AMDGPU::V_FMAAK_F16_t16
4005 : AMDGPU::V_FMAAK_F16_fake16
4006 : AMDGPU::V_FMAAK_F16;
4007 case AMDGPU::V_FMAC_F64_e32:
4008 case AMDGPU::V_FMAC_F64_e64:
4009 case AMDGPU::V_FMA_F64_e64:
4010 return AMDGPU::V_FMAAK_F64;
4018 case AMDGPU::V_MAC_F16_e32:
4019 case AMDGPU::V_MAC_F16_e64:
4020 case AMDGPU::V_MAD_F16_e64:
4021 return AMDGPU::V_MADMK_F16;
4022 case AMDGPU::V_MAC_F32_e32:
4023 case AMDGPU::V_MAC_F32_e64:
4024 case AMDGPU::V_MAD_F32_e64:
4025 return AMDGPU::V_MADMK_F32;
4026 case AMDGPU::V_FMAC_F32_e32:
4027 case AMDGPU::V_FMAC_F32_e64:
4028 case AMDGPU::V_FMA_F32_e64:
4029 return AMDGPU::V_FMAMK_F32;
4030 case AMDGPU::V_FMAC_F16_e32:
4031 case AMDGPU::V_FMAC_F16_e64:
4032 case AMDGPU::V_FMAC_F16_t16_e64:
4033 case AMDGPU::V_FMAC_F16_fake16_e64:
4034 case AMDGPU::V_FMAC_F16_t16_e32:
4035 case AMDGPU::V_FMAC_F16_fake16_e32:
4036 case AMDGPU::V_FMA_F16_e64:
4037 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4038 ? AMDGPU::V_FMAMK_F16_t16
4039 : AMDGPU::V_FMAMK_F16_fake16
4040 : AMDGPU::V_FMAMK_F16;
4041 case AMDGPU::V_FMAC_F64_e32:
4042 case AMDGPU::V_FMAC_F64_e64:
4043 case AMDGPU::V_FMA_F64_e64:
4044 return AMDGPU::V_FMAMK_F64;
4056 const bool HasMultipleUses = !
MRI->hasOneNonDBGUse(Reg);
4058 assert(!
DefMI.getOperand(0).getSubReg() &&
"Expected SSA form");
4061 if (
Opc == AMDGPU::COPY) {
4062 assert(!
UseMI.getOperand(0).getSubReg() &&
"Expected SSA form");
4069 if (HasMultipleUses) {
4072 unsigned ImmDefSize = RI.getRegSizeInBits(*
MRI->getRegClass(Reg));
4075 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)
4083 if (ImmDefSize == 32 &&
4088 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
4089 RI.getSubRegIdxSize(UseSubReg) == 16;
4092 if (RI.hasVGPRs(DstRC))
4095 if (DstReg.
isVirtual() && UseSubReg != AMDGPU::lo16)
4101 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
4108 for (
unsigned MovOp :
4109 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
4110 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
4118 MovDstRC = RI.getMatchingSuperRegClass(MovDstRC, DstRC, AMDGPU::lo16);
4122 if (MovDstPhysReg) {
4126 RI.getMatchingSuperReg(MovDstPhysReg, AMDGPU::lo16, MovDstRC);
4133 if (MovDstPhysReg) {
4134 if (!MovDstRC->
contains(MovDstPhysReg))
4136 }
else if (!
MRI->constrainRegClass(DstReg, MovDstRC)) {
4150 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType) &&
4158 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
4162 UseMI.getOperand(0).setSubReg(AMDGPU::NoSubRegister);
4164 UseMI.getOperand(0).setReg(MovDstPhysReg);
4169 UseMI.setDesc(NewMCID);
4170 UseMI.getOperand(1).ChangeToImmediate(*SubRegImm);
4171 UseMI.addImplicitDefUseOperands(*MF);
4175 if (HasMultipleUses)
4178 if (
Opc == AMDGPU::V_MAD_F32_e64 ||
Opc == AMDGPU::V_MAC_F32_e64 ||
4179 Opc == AMDGPU::V_MAD_F16_e64 ||
Opc == AMDGPU::V_MAC_F16_e64 ||
4180 Opc == AMDGPU::V_FMA_F32_e64 ||
Opc == AMDGPU::V_FMAC_F32_e64 ||
4181 Opc == AMDGPU::V_FMA_F16_e64 ||
Opc == AMDGPU::V_FMAC_F16_e64 ||
4182 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4183 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
Opc == AMDGPU::V_FMA_F64_e64 ||
4184 Opc == AMDGPU::V_FMAC_F64_e64) {
4193 int Src0Idx = getNamedOperandIdx(
UseMI.getOpcode(), AMDGPU::OpName::src0);
4204 auto CopyRegOperandToNarrowerRC =
4207 if (!
MI.getOperand(OpNo).isReg())
4211 if (RI.getCommonSubClass(RC, NewRC) != NewRC)
4214 BuildMI(*
MI.getParent(),
MI.getIterator(),
MI.getDebugLoc(),
4215 get(AMDGPU::COPY), Tmp)
4217 MI.getOperand(OpNo).setReg(Tmp);
4221 if ((Src0->isReg() && Src0->getReg() == Reg) ||
4222 (Src1->isReg() && Src1->getReg() == Reg)) {
4224 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
4225 if (!RegSrc->
isReg())
4227 if (RI.isSGPRClass(
MRI->getRegClass(RegSrc->
getReg())) &&
4228 ST.getConstantBusLimit(
Opc) < 2)
4231 if (!Src2->isReg() || RI.isSGPRClass(
MRI->getRegClass(Src2->getReg())))
4243 if (Def && Def->isMoveImmediate() &&
4252 Imm, RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
4258 unsigned SrcSubReg = RegSrc->
getSubReg();
4259 Src0->setReg(SrcReg);
4260 Src0->setSubReg(SrcSubReg);
4261 Src0->setIsKill(RegSrc->
isKill());
4263 if (
Opc == AMDGPU::V_MAC_F32_e64 ||
Opc == AMDGPU::V_MAC_F16_e64 ||
4264 Opc == AMDGPU::V_FMAC_F32_e64 ||
Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4265 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4266 Opc == AMDGPU::V_FMAC_F16_e64 ||
Opc == AMDGPU::V_FMAC_F64_e64)
4267 UseMI.untieRegOperand(
4268 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2));
4270 Src1->ChangeToImmediate(*SubRegImm);
4275 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||
4276 NewOpc == AMDGPU::V_FMAMK_F16_fake16) {
4280 UseMI.getDebugLoc(),
get(AMDGPU::COPY),
4281 UseMI.getOperand(0).getReg())
4283 UseMI.getOperand(0).setReg(Tmp);
4284 CopyRegOperandToNarrowerRC(
UseMI, 1, NewRC);
4285 CopyRegOperandToNarrowerRC(
UseMI, 3, NewRC);
4288 bool DeleteDef =
MRI->use_nodbg_empty(Reg);
4290 DefMI.eraseFromParent();
4296 if (Src2->isReg() && Src2->getReg() == Reg) {
4297 if (ST.getConstantBusLimit(
Opc) < 2) {
4300 bool Src0Inlined =
false;
4301 if (Src0->isReg()) {
4306 if (Def && Def->isMoveImmediate() &&
4308 MRI->hasOneNonDBGUse(Src0->getReg())) {
4309 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
4311 }
else if (ST.getConstantBusLimit(
Opc) <= 1 &&
4312 RI.isSGPRReg(*
MRI, Src0->getReg())) {
4318 if (Src1->isReg() && !Src0Inlined) {
4321 if (Def && Def->isMoveImmediate() &&
4323 MRI->hasOneNonDBGUse(Src1->getReg()) && commuteInstruction(
UseMI))
4324 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
4325 else if (RI.isSGPRReg(*
MRI, Src1->getReg()))
4338 if (
Opc == AMDGPU::V_MAC_F32_e64 ||
Opc == AMDGPU::V_MAC_F16_e64 ||
4339 Opc == AMDGPU::V_FMAC_F32_e64 ||
Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4340 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4341 Opc == AMDGPU::V_FMAC_F16_e64 ||
Opc == AMDGPU::V_FMAC_F64_e64)
4342 UseMI.untieRegOperand(
4343 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2));
4345 const std::optional<int64_t> SubRegImm =
4349 Src2->ChangeToImmediate(*SubRegImm);
4355 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||
4356 NewOpc == AMDGPU::V_FMAAK_F16_fake16) {
4360 UseMI.getDebugLoc(),
get(AMDGPU::COPY),
4361 UseMI.getOperand(0).getReg())
4363 UseMI.getOperand(0).setReg(Tmp);
4364 CopyRegOperandToNarrowerRC(
UseMI, 1, NewRC);
4365 CopyRegOperandToNarrowerRC(
UseMI, 2, NewRC);
4374 AMDGPU::getNamedOperandIdx(
UseMI.getOpcode(), AMDGPU::OpName::src0);
4378 bool DeleteDef =
MRI->use_nodbg_empty(Reg);
4380 DefMI.eraseFromParent();
4392 if (BaseOps1.
size() != BaseOps2.
size())
4394 for (
size_t I = 0,
E = BaseOps1.
size();
I <
E; ++
I) {
4395 if (!BaseOps1[
I]->isIdenticalTo(*BaseOps2[
I]))
4403 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
4404 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
4405 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4407 LowOffset + (int)LowWidth.
getValue() <= HighOffset;
4410bool SIInstrInfo::checkInstOffsetsDoNotOverlap(
const MachineInstr &MIa,
4413 int64_t Offset0, Offset1;
4416 bool Offset0IsScalable, Offset1IsScalable;
4430 LocationSize Width0 = MIa.
memoperands().front()->getSize();
4431 LocationSize Width1 = MIb.
memoperands().front()->getSize();
4438 "MIa must load from or modify a memory location");
4440 "MIb must load from or modify a memory location");
4458 if (IsLdStIdxA || IsLdStIdxB) {
4459 if (IsLdStIdxA && IsLdStIdxB)
4460 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4471 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4478 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4488 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4502 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4513 case AMDGPU::V_MAC_F16_e32:
4514 case AMDGPU::V_MAC_F16_e64:
4515 return AMDGPU::V_MAD_F16_e64;
4516 case AMDGPU::V_MAC_F32_e32:
4517 case AMDGPU::V_MAC_F32_e64:
4518 return AMDGPU::V_MAD_F32_e64;
4519 case AMDGPU::V_MAC_LEGACY_F32_e32:
4520 case AMDGPU::V_MAC_LEGACY_F32_e64:
4521 return AMDGPU::V_MAD_LEGACY_F32_e64;
4522 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4523 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4524 return AMDGPU::V_FMA_LEGACY_F32_e64;
4525 case AMDGPU::V_FMAC_F16_e32:
4526 case AMDGPU::V_FMAC_F16_e64:
4527 case AMDGPU::V_FMAC_F16_t16_e64:
4528 case AMDGPU::V_FMAC_F16_fake16_e64:
4529 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4530 ? AMDGPU::V_FMA_F16_gfx9_t16_e64
4531 : AMDGPU::V_FMA_F16_gfx9_fake16_e64
4532 : AMDGPU::V_FMA_F16_gfx9_e64;
4533 case AMDGPU::V_FMAC_F32_e32:
4534 case AMDGPU::V_FMAC_F32_e64:
4535 return AMDGPU::V_FMA_F32_e64;
4536 case AMDGPU::V_FMAC_F64_e32:
4537 case AMDGPU::V_FMAC_F64_e64:
4538 return AMDGPU::V_FMA_F64_e64;
4557 if (
MI.isBundle()) {
4560 if (
MI.getBundleSize() != 1)
4562 CandidateMI =
MI.getNextNode();
4566 MachineInstr *NewMI = convertToThreeAddressImpl(*CandidateMI, U);
4570 if (
MI.isBundle()) {
4575 MI.untieRegOperand(MO.getOperandNo());
4582 if (Def.isEarlyClobber() && Def.isReg() &&
4587 auto UpdateDefIndex = [&](
LiveRange &LR) {
4588 auto *S = LR.find(OldIndex);
4589 if (S != LR.end() && S->start == OldIndex) {
4590 assert(S->valno && S->valno->def == OldIndex);
4591 S->start = NewIndex;
4592 S->valno->def = NewIndex;
4596 for (
auto &SR : LI.subranges())
4606 "expected a tied def in the two-address form");
4611 SlotIndex OldEnd = Idx.getRegSlot(
true);
4613 auto UpdateUseIndex = [&](
LiveRange &LR) {
4615 if (S && S->
end == OldEnd)
4626 if (U.RemoveMIUse) {
4629 Register DefReg = U.RemoveMIUse->getOperand(0).getReg();
4631 if (
MRI.hasOneNonDBGUse(DefReg)) {
4633 U.RemoveMIUse->setDesc(
get(AMDGPU::IMPLICIT_DEF));
4634 U.RemoveMIUse->getOperand(0).setIsDead(
true);
4635 for (
unsigned I = U.RemoveMIUse->getNumOperands() - 1;
I != 0; --
I)
4636 U.RemoveMIUse->removeOperand(
I);
4639 if (
MI.isBundle()) {
4643 if (MO.isReg() && MO.getReg() == DefReg) {
4644 assert(MO.getSubReg() == 0 &&
4645 "tied sub-registers in bundles currently not supported");
4646 MI.removeOperand(MO.getOperandNo());
4661 Register DummyReg =
MRI.cloneVirtualRegister(DefReg);
4663 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4664 MIOp.setIsUndef(
true);
4665 MIOp.setReg(DummyReg);
4669 if (
MI.isBundle()) {
4673 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4674 MIOp.setIsUndef(
true);
4675 MIOp.setReg(DummyReg);
4688 return MI.isBundle() ? &
MI : NewMI;
4693 ThreeAddressUpdates &U)
const {
4695 unsigned Opc =
MI.getOpcode();
4699 if (NewMFMAOpc != -1) {
4702 for (
unsigned I = 0, E =
MI.getNumExplicitOperands();
I != E; ++
I)
4703 MIB.
add(
MI.getOperand(
I));
4711 for (
unsigned I = 0,
E =
MI.getNumExplicitOperands();
I !=
E; ++
I)
4716 assert(
Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
4717 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
4718 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "
4722 bool IsF64 =
Opc == AMDGPU::V_FMAC_F64_e32 ||
Opc == AMDGPU::V_FMAC_F64_e64;
4723 bool IsLegacy =
Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
4724 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
4725 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
4726 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
4727 bool Src0Literal =
false;
4732 case AMDGPU::V_MAC_F16_e64:
4733 case AMDGPU::V_FMAC_F16_e64:
4734 case AMDGPU::V_FMAC_F16_t16_e64:
4735 case AMDGPU::V_FMAC_F16_fake16_e64:
4736 case AMDGPU::V_MAC_F32_e64:
4737 case AMDGPU::V_MAC_LEGACY_F32_e64:
4738 case AMDGPU::V_FMAC_F32_e64:
4739 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4740 case AMDGPU::V_FMAC_F64_e64:
4742 case AMDGPU::V_MAC_F16_e32:
4743 case AMDGPU::V_FMAC_F16_e32:
4744 case AMDGPU::V_MAC_F32_e32:
4745 case AMDGPU::V_MAC_LEGACY_F32_e32:
4746 case AMDGPU::V_FMAC_F32_e32:
4747 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4748 case AMDGPU::V_FMAC_F64_e32: {
4749 int Src0Idx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(),
4750 AMDGPU::OpName::src0);
4751 const MachineOperand *
Src0 = &
MI.getOperand(Src0Idx);
4752 if (!
Src0->isReg() && !
Src0->isImm())
4762 MachineInstrBuilder MIB;
4765 const MachineOperand *Src0Mods =
4768 const MachineOperand *Src1Mods =
4771 const MachineOperand *Src2Mods =
4777 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&
4778 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&
4780 (ST.getConstantBusLimit(
Opc) > 1 || !
Src0->isReg() ||
4782 MachineInstr *
DefMI =
nullptr;
4784 std::optional<int64_t> ImmOpt;
4819 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
4835 if (Src0Literal && !ST.hasVOP3Literal())
4863 switch (
MI.getOpcode()) {
4864 case AMDGPU::S_SET_GPR_IDX_ON:
4865 case AMDGPU::S_SET_GPR_IDX_MODE:
4866 case AMDGPU::S_SET_GPR_IDX_OFF:
4884 if (
MI.isTerminator() ||
MI.isPosition())
4888 if (
MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4891 if (
MI.getOpcode() == AMDGPU::SCHED_BARRIER &&
MI.getOperand(0).getImm() == 0)
4897 return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
4898 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4899 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4900 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4901 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
4906 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
4907 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
4908 Opcode == AMDGPU::DS_SUB_GS_REG_RTN ||
isGWS(Opcode);
4922 if (
MI.getMF()->getFunction().hasFnAttribute(
"amdgpu-no-flat-scratch-init"))
4927 if (
MI.memoperands_empty())
4932 unsigned AS = Memop->getAddrSpace();
4933 if (AS == AMDGPUAS::FLAT_ADDRESS) {
4934 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
4935 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
4936 *MD, AMDGPUAS::PRIVATE_ADDRESS);
4951 if (
MI.memoperands_empty())
4960 unsigned AS = Memop->getAddrSpace();
4970 bool TgSplit)
const {
4983 if (
MI.memoperands_empty())
4988 unsigned AS = Memop->getAddrSpace();
5006 switch (
MI.getOpcode()) {
5008 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8:
5009 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay:
5010 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_SADDR:
5011 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32:
5012 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR:
5013 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64:
5014 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR:
5015 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128:
5016 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR:
5018 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8:
5019 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8_SADDR:
5020 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32:
5021 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR:
5022 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64:
5023 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64_SADDR:
5024 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128:
5025 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128_SADDR:
5027 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8:
5028 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR:
5029 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32:
5030 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR:
5031 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64:
5032 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR:
5033 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128:
5034 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR:
5049 unsigned Opcode =
MI.getOpcode();
5064 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
5065 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
5066 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT ||
5067 Opcode == AMDGPU::S_SETHALT)
5070 if (
MI.isCall() ||
MI.isInlineAsm())
5076 if (ST.hasVPermPk16Hazard() &&
isVPermPk16(Opcode))
5092 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
5093 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
5094 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
5095 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
5103 if (
MI.isMetaInstruction())
5107 if (
MI.isCopyLike()) {
5108 if (!RI.isSGPRReg(
MRI,
MI.getOperand(0).getReg()))
5112 return MI.readsRegister(AMDGPU::EXEC, &RI);
5123 return !
isSALU(
MI) ||
MI.readsRegister(AMDGPU::EXEC, &RI);
5127 switch (
Imm.getBitWidth()) {
5133 ST.hasInv2PiInlineImm());
5136 ST.hasInv2PiInlineImm());
5138 return ST.has16BitInsts() &&
5140 ST.hasInv2PiInlineImm());
5147 APInt IntImm =
Imm.bitcastToAPInt();
5149 bool HasInv2Pi = ST.hasInv2PiInlineImm();
5157 return ST.has16BitInsts() &&
5160 return ST.has16BitInsts() &&
5170 switch (OperandType) {
5180 int32_t Trunc =
static_cast<int32_t
>(
Imm);
5224 int16_t Trunc =
static_cast<int16_t
>(
Imm);
5225 return ST.has16BitInsts() &&
5234 int16_t Trunc =
static_cast<int16_t
>(
Imm);
5235 return ST.has16BitInsts() &&
5287 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
5293 return ST.hasVOP3Literal();
5297 int64_t ImmVal)
const {
5299 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5300 if (Src1Idx != -1 &&
isDPP(
Opc) && !ST.hasDPPSrc1SGPR() &&
5301 OpNo ==
static_cast<unsigned>(Src1Idx))
5306 if (
isMAI(InstDesc) && ST.hasMFMAInlineLiteralBug() &&
5307 OpNo == (
unsigned)AMDGPU::getNamedOperandIdx(InstDesc.
getOpcode(),
5308 AMDGPU::OpName::src2))
5311 if (ST.hasBF16InlineConstFromUpperFP32() &&
isVOP1(
Opc)) {
5318 return RI.opCanUseInlineConstant(OpInfo.OperandType);
5330 "unexpected imm-like operand kind");
5343 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
5363 return Op32 != -1 &&
TII.isVOPC(Op32);
5368 unsigned Depth)
const {
5369 assert(
MRI.isSSA() &&
"isMaskedByExec requires SSA form");
5378 constexpr unsigned MaxDepth = 6;
5379 if (
Depth >= MaxDepth || !Reg.isVirtual())
5385 if (!Def || Def->getParent() !=
MBB)
5393 auto Recurse = [&](
unsigned OpIdx) {
5399 unsigned Opc = Def->getOpcode();
5400 if (
Opc == AMDGPU::COPY && Recurse(1))
5402 if (
Opc == LMC.
AndOpc && (Recurse(1) || Recurse(2)))
5422 AMDGPU::OpName
OpName)
const {
5424 return Mods && Mods->
getImm();
5437 switch (
MI.getOpcode()) {
5438 default:
return false;
5440 case AMDGPU::V_ADDC_U32_e64:
5441 case AMDGPU::V_SUBB_U32_e64:
5442 case AMDGPU::V_SUBBREV_U32_e64: {
5445 if (!Src1->isReg() || !RI.isVGPR(
MRI, Src1->getReg()))
5450 case AMDGPU::V_MAC_F16_e64:
5451 case AMDGPU::V_MAC_F32_e64:
5452 case AMDGPU::V_MAC_LEGACY_F32_e64:
5453 case AMDGPU::V_FMAC_F16_e64:
5454 case AMDGPU::V_FMAC_F16_t16_e64:
5455 case AMDGPU::V_FMAC_F16_fake16_e64:
5456 case AMDGPU::V_FMAC_F32_e64:
5457 case AMDGPU::V_FMAC_F64_e64:
5458 case AMDGPU::V_FMAC_LEGACY_F32_e64:
5459 if (!Src2->isReg() || !RI.isVGPR(
MRI, Src2->getReg()) ||
5464 case AMDGPU::V_CNDMASK_B32_e64:
5470 if (Src1 && (!Src1->isReg() || !RI.isVGPR(
MRI, Src1->getReg()) ||
5483 if (Src0 && Src0->isImm()) {
5486 get(Op32), AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src0),
5508 (
Use.getReg() == AMDGPU::VCC ||
Use.getReg() == AMDGPU::VCC_LO)) {
5517 unsigned Op32)
const {
5531 Inst32.
add(
MI.getOperand(
I));
5535 int Idx =
MI.getNumExplicitDefs();
5537 int OpTy =
MI.getDesc().operands()[Idx++].OperandType;
5542 if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2) == -1) {
5562 if (OldSDst && OldSDst->
isDead()) {
5565 NewVCC->setIsDead();
5574 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)
5582 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;
5585 return AMDGPU::SReg_32RegClass.contains(Reg) ||
5586 AMDGPU::SReg_64RegClass.contains(Reg);
5592 return Reg.
isVirtual() ? RI.isSGPRClass(
MRI.getRegClass(Reg))
5604 return Reg.
isVirtual() ? RI.isSGPRClass(
MRI.getRegClass(Reg))
5614 switch (MO.getReg()) {
5616 case AMDGPU::VCC_LO:
5617 case AMDGPU::VCC_HI:
5619 case AMDGPU::FLAT_SCR:
5632 switch (
MI.getOpcode()) {
5633 case AMDGPU::V_READLANE_B32:
5634 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
5635 case AMDGPU::V_WRITELANE_B32:
5636 case AMDGPU::SI_SPILL_S32_TO_VGPR:
5643 if (
MI.isPreISelOpcode() ||
5644 SIInstrInfo::isGenericOpcode(
MI.getOpcode()) ||
5662 return SubReg.
getSubReg() != AMDGPU::NoSubRegister &&
5673 if (RI.isVectorRegister(
MRI, SrcReg) && RI.isSGPRReg(
MRI, DstReg)) {
5674 ErrInfo =
"illegal copy from vector register to SGPR";
5692 if (!
MRI.isSSA() &&
MI.isCopy())
5693 return verifyCopy(
MI,
MRI, ErrInfo);
5695 if (SIInstrInfo::isGenericOpcode(Opcode) && !
MI.isInlineAsm())
5698 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
5699 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
5700 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
5702 if (Src0Idx == -1) {
5704 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X);
5705 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X);
5706 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y);
5707 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y);
5712 if (!
Desc.isVariadic() &&
5713 Desc.getNumOperands() !=
MI.getNumExplicitOperands()) {
5714 ErrInfo =
"Instruction has wrong number of operands.";
5718 if (
MI.isInlineAsm()) {
5731 if (!Reg.isVirtual() && !RC->
contains(Reg)) {
5732 ErrInfo =
"inlineasm operand has incorrect register class.";
5740 if (
isImage(
MI) &&
MI.memoperands_empty() &&
MI.mayLoadOrStore()) {
5741 ErrInfo =
"missing memory operand from image instruction.";
5746 for (
int i = 0, e =
Desc.getNumOperands(); i != e; ++i) {
5749 ErrInfo =
"FPImm Machine Operands are not supported. ISel should bitcast "
5750 "all fp values to integers.";
5756 switch (OpInfo.OperandType) {
5758 if (
MI.getOperand(i).isImm() ||
MI.getOperand(i).isGlobal()) {
5759 ErrInfo =
"Illegal immediate value for operand.";
5791 ErrInfo =
"Illegal immediate value for operand.";
5800 if (ST.has64BitLiterals() &&
Desc.getSize() != 4 && MO.
isImm() &&
5803 OpInfo.OperandType ==
5805 ErrInfo =
"illegal 64-bit immediate value for operand.";
5812 ErrInfo =
"Expected inline constant for operand.";
5826 if (!
MI.getOperand(i).isImm() && !
MI.getOperand(i).isFI()) {
5827 ErrInfo =
"Expected immediate, but got non-immediate";
5836 if (OpInfo.isGenericType())
5844 if (!ST.hasSDWA()) {
5845 ErrInfo =
"SDWA is not supported on this target";
5849 for (
auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,
5850 AMDGPU::OpName::dst_sel}) {
5856 ErrInfo =
"Invalid SDWA selection";
5861 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
5863 for (
int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
5868 if (!ST.hasSDWAScalar()) {
5870 if (!MO.
isReg() || !RI.hasVGPRs(RI.getRegClassForReg(
MRI, MO.
getReg()))) {
5871 ErrInfo =
"Only VGPRs allowed as operands in SDWA instructions on VI";
5878 "Only reg allowed as operands in SDWA instructions on GFX9+";
5884 if (!ST.hasSDWAOmod()) {
5887 if (OMod !=
nullptr &&
5889 ErrInfo =
"OMod not allowed in SDWA instructions on VI";
5894 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
5895 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
5896 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
5897 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
5900 unsigned Mods = Src0ModsMO->
getImm();
5903 ErrInfo =
"sext, abs and neg are not allowed on this instruction";
5909 if (
isVOPC(BasicOpcode)) {
5910 if (!ST.hasSDWASdst() && DstIdx != -1) {
5913 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
5914 ErrInfo =
"Only VCC allowed as dst in SDWA instructions on VI";
5917 }
else if (!ST.hasSDWAOutModsVOPC()) {
5920 if (Clamp && (!Clamp->
isImm() || Clamp->
getImm() != 0)) {
5921 ErrInfo =
"Clamp not allowed in VOPC SDWA instructions on VI";
5927 if (OMod && (!OMod->
isImm() || OMod->
getImm() != 0)) {
5928 ErrInfo =
"OMod not allowed in VOPC SDWA instructions on VI";
5935 if (DstUnused && DstUnused->isImm() &&
5938 if (!Dst.isReg() || !Dst.isTied()) {
5939 ErrInfo =
"Dst register should have tied register";
5944 MI.getOperand(
MI.findTiedOperandIdx(DstIdx));
5947 "Dst register should be tied to implicit use of preserved register";
5951 ErrInfo =
"Dst register should use same physical register as preserved";
5957 if (
isDPP(
MI) && !ST.hasDPPSrc1SGPR() && Src1Idx != -1) {
5960 ErrInfo =
"DPP src1 cannot be SGPR on this subtarget";
5963 if (Src1MO.
isImm()) {
5964 ErrInfo =
"DPP src1 cannot be an immediate on this subtarget";
5970 if (
isImage(Opcode) && !
MI.mayStore()) {
5975 uint64_t DMaskImm = DMask->
getImm();
5982 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
5990 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
5994 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
5995 if (RegCount > DstSize) {
5996 ErrInfo =
"Image instruction returns too many registers for dst "
6006 Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
6007 unsigned ConstantBusCount = 0;
6008 bool UsesLiteral =
false;
6011 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);
6015 LiteralVal = &
MI.getOperand(ImmIdx);
6024 for (
int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
6035 }
else if (!MO.
isFI()) {
6042 ErrInfo =
"VOP2/VOP3 instruction uses more than one literal";
6052 if (
llvm::all_of(SGPRsUsed, [
this, SGPRUsed](
unsigned SGPR) {
6053 return !RI.regsOverlap(SGPRUsed, SGPR);
6062 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
6063 Opcode != AMDGPU::V_WRITELANE_B32) {
6064 ErrInfo =
"VOP* instruction violates constant bus restriction";
6068 if (
isVOP3(
MI) && UsesLiteral && !ST.hasVOP3Literal()) {
6069 ErrInfo =
"VOP3 instruction uses literal";
6076 if (
Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
6077 unsigned SGPRCount = 0;
6080 for (
int OpIdx : {Src0Idx, Src1Idx}) {
6088 if (MO.
getReg() != SGPRUsed)
6093 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
6094 ErrInfo =
"WRITELANE instruction violates constant bus restriction";
6100 if (Opcode == AMDGPU::ASYNCMARK || Opcode == AMDGPU::WAIT_ASYNCMARK) {
6101 int64_t Mask =
MI.getOperand(
MI.getNumExplicitOperands() - 1).getImm();
6103 ErrInfo =
"invalid asyncmark stage mask";
6113 Offset.getImm() + LdSt->getBitWidth() / 32 >
6114 AMDGPU::VGPR_32RegClass.getNumRegs()) {
6115 ErrInfo =
"invalid VGPR address space offset";
6121 if (
Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
6122 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
6126 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
6129 ErrInfo =
"v_div_scale_{f32|f64} require src0 = src1 or src2";
6139 ErrInfo =
"ABS not allowed in VOP3B instructions";
6151 !Src0.isIdenticalTo(Src1)) {
6152 ErrInfo =
"SOP2/SOPC instruction requires too many immediate constants";
6159 if (
Desc.isBranch()) {
6161 ErrInfo =
"invalid branch target for SOPK instruction";
6165 uint64_t
Imm =
Op->getImm();
6168 ErrInfo =
"invalid immediate for SOPK instruction";
6173 ErrInfo =
"invalid immediate for SOPK instruction";
6180 if (
Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
6181 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
6182 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
6183 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
6184 const bool IsDst =
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
6185 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
6187 const unsigned StaticNumOps =
6188 Desc.getNumOperands() +
Desc.implicit_uses().size();
6189 const unsigned NumImplicitOps = IsDst ? 2 : 1;
6195 if (
MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
6196 ErrInfo =
"missing implicit register operands";
6202 if (!Dst->isUse()) {
6203 ErrInfo =
"v_movreld_b32 vdst should be a use operand";
6208 if (!
MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
6209 UseOpIdx != StaticNumOps + 1) {
6210 ErrInfo =
"movrel implicit operands should be tied";
6217 =
MI.getOperand(StaticNumOps + NumImplicitOps - 1);
6219 !
isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
6220 ErrInfo =
"src0 should be subreg of implicit vector use";
6228 if (!
MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
6229 ErrInfo =
"VALU instruction does not implicitly read exec mask";
6235 if (
MI.mayStore() &&
6240 if (Soff && Soff->
getReg() != AMDGPU::M0) {
6241 ErrInfo =
"scalar stores must use m0 as offset register";
6247 if (
isFLAT(
MI) && !ST.hasFlatInstOffsets()) {
6249 if (
Offset->getImm() != 0) {
6250 ErrInfo =
"subtarget does not support offsets in flat instructions";
6255 if (
isDS(
MI) && !ST.hasGDS()) {
6257 if (GDSOp && GDSOp->
getImm() != 0) {
6258 ErrInfo =
"GDS is not supported on this subtarget";
6266 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
6267 AMDGPU::OpName::vaddr0);
6268 AMDGPU::OpName RSrcOpName =
6269 isMIMG(
MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
6270 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, RSrcOpName);
6278 ErrInfo =
"dim is out of range";
6283 if (ST.hasR128A16()) {
6285 IsA16 = R128A16->
getImm() != 0;
6286 }
else if (ST.hasA16()) {
6288 IsA16 = A16->
getImm() != 0;
6291 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
6293 unsigned AddrWords =
6296 unsigned VAddrWords;
6298 VAddrWords = RsrcIdx - VAddr0Idx;
6299 if (ST.hasPartialNSAEncoding() &&
6301 unsigned LastVAddrIdx = RsrcIdx - 1;
6302 VAddrWords +=
getOpSize(
MI, LastVAddrIdx) / 4 - 1;
6310 if (VAddrWords != AddrWords) {
6312 <<
" but got " << VAddrWords <<
"\n");
6313 ErrInfo =
"bad vaddr size";
6323 unsigned DC = DppCt->
getImm();
6324 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
6325 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
6326 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
6327 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
6328 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
6329 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
6330 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
6331 ErrInfo =
"Invalid dpp_ctrl value";
6334 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
6335 !ST.hasDPPWavefrontShifts()) {
6336 ErrInfo =
"Invalid dpp_ctrl value: "
6337 "wavefront shifts are not supported on GFX10+";
6340 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
6341 !ST.hasDPPBroadcasts()) {
6342 ErrInfo =
"Invalid dpp_ctrl value: "
6343 "broadcasts are not supported on GFX10+";
6346 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
6348 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
6349 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
6350 !ST.hasGFX90AInsts()) {
6351 ErrInfo =
"Invalid dpp_ctrl value: "
6352 "row_newbroadcast/row_share is not supported before "
6356 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
6357 ErrInfo =
"Invalid dpp_ctrl value: "
6358 "row_share and row_xmask are not supported before GFX10";
6363 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
6365 ST.hasFeature(AMDGPU::FeatureDPALU_DPP) &&
6367 ErrInfo =
"Invalid dpp_ctrl value: "
6368 "DP ALU dpp only support row_newbcast";
6375 AMDGPU::OpName DataName =
6376 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;
6382 if (!ST.hasGFX90AInsts()) {
6383 if ((Dst && RI.isAGPR(
MRI, Dst->getReg())) ||
6385 (Data2 && RI.isAGPR(
MRI, Data2->
getReg()))) {
6386 ErrInfo =
"Invalid register class: "
6387 "agpr loads and stores not supported on this GPU";
6393 if (ST.needsAlignedVGPRs()) {
6394 const auto isAlignedReg = [&
MI, &
MRI,
this](AMDGPU::OpName
OpName) ->
bool {
6399 if (Reg.isPhysical())
6400 return !(RI.getHWRegIndex(Reg) & 1);
6402 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
6403 !(RI.getChannelFromSubReg(
Op->getSubReg()) & 1);
6407 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
6408 ErrInfo =
"Subtarget requires even aligned vector registers "
6409 "for vaddr operand of image instructions";
6415 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {
6417 if (Src->isReg() && RI.isSGPRReg(
MRI, Src->getReg())) {
6418 ErrInfo =
"Invalid register class: "
6419 "v_accvgpr_write with an SGPR is not supported on this GPU";
6424 if (
Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
6427 ErrInfo =
"pseudo expects only physical SGPRs";
6434 if (!ST.hasScaleOffset()) {
6435 ErrInfo =
"subtarget does not support offset scaling";
6439 ErrInfo =
"instruction does not support offset scaling";
6447 for (
unsigned I = 0;
I < 3; ++
I) {
6453 if (ST.hasFlatScratchHiInB64InstHazard() &&
isSALU(
MI) &&
6454 MI.readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI,
nullptr)) {
6456 if ((Dst && RI.getRegClassForReg(
MRI, Dst->getReg()) ==
6457 &AMDGPU::SReg_64RegClass) ||
6458 Opcode == AMDGPU::S_BITCMP0_B64 || Opcode == AMDGPU::S_BITCMP1_B64) {
6459 ErrInfo =
"Instruction cannot read flat_scratch_base_hi";
6466 ErrInfo =
"GFX950 v_pk_{add,mul,fma}_f32 has an invalid op_sel.";
6474 if (
MI.getOpcode() == AMDGPU::S_MOV_B32) {
6476 return MI.getOperand(1).isReg() || RI.isAGPR(
MRI,
MI.getOperand(0).getReg())
6478 : AMDGPU::V_MOV_B32_e32;
6488 default:
return AMDGPU::INSTRUCTION_LIST_END;
6489 case AMDGPU::REG_SEQUENCE:
return AMDGPU::REG_SEQUENCE;
6490 case AMDGPU::COPY:
return AMDGPU::COPY;
6491 case AMDGPU::PHI:
return AMDGPU::PHI;
6492 case AMDGPU::INSERT_SUBREG:
return AMDGPU::INSERT_SUBREG;
6493 case AMDGPU::WQM:
return AMDGPU::WQM;
6494 case AMDGPU::SOFT_WQM:
return AMDGPU::SOFT_WQM;
6495 case AMDGPU::STRICT_WWM:
return AMDGPU::STRICT_WWM;
6496 case AMDGPU::STRICT_WQM:
return AMDGPU::STRICT_WQM;
6497 case AMDGPU::S_ADD_I32:
6498 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
6499 case AMDGPU::S_ADDC_U32:
6500 return AMDGPU::V_ADDC_U32_e32;
6501 case AMDGPU::S_SUB_I32:
6502 return ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
6505 case AMDGPU::S_ADD_U32:
6506 return AMDGPU::V_ADD_CO_U32_e32;
6507 case AMDGPU::S_SUB_U32:
6508 return AMDGPU::V_SUB_CO_U32_e32;
6509 case AMDGPU::S_ADD_U64_PSEUDO:
6510 return AMDGPU::V_ADD_U64_PSEUDO;
6511 case AMDGPU::S_SUB_U64_PSEUDO:
6512 return AMDGPU::V_SUB_U64_PSEUDO;
6513 case AMDGPU::S_SUBB_U32:
return AMDGPU::V_SUBB_U32_e32;
6514 case AMDGPU::S_MUL_I32:
return AMDGPU::V_MUL_LO_U32_e64;
6515 case AMDGPU::S_MUL_HI_U32:
return AMDGPU::V_MUL_HI_U32_e64;
6516 case AMDGPU::S_MUL_HI_I32:
return AMDGPU::V_MUL_HI_I32_e64;
6517 case AMDGPU::S_AND_B32:
return AMDGPU::V_AND_B32_e64;
6518 case AMDGPU::S_OR_B32:
return AMDGPU::V_OR_B32_e64;
6519 case AMDGPU::S_XOR_B32:
return AMDGPU::V_XOR_B32_e64;
6520 case AMDGPU::S_XNOR_B32:
6521 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
6522 case AMDGPU::S_MIN_I32:
return AMDGPU::V_MIN_I32_e64;
6523 case AMDGPU::S_MIN_U32:
return AMDGPU::V_MIN_U32_e64;
6524 case AMDGPU::S_MAX_I32:
return AMDGPU::V_MAX_I32_e64;
6525 case AMDGPU::S_MAX_U32:
return AMDGPU::V_MAX_U32_e64;
6526 case AMDGPU::S_ASHR_I32:
return AMDGPU::V_ASHR_I32_e32;
6527 case AMDGPU::S_ASHR_I64:
return AMDGPU::V_ASHR_I64_e64;
6528 case AMDGPU::S_LSHL_B32:
return AMDGPU::V_LSHL_B32_e32;
6529 case AMDGPU::S_LSHL_B64:
return AMDGPU::V_LSHL_B64_e64;
6530 case AMDGPU::S_LSHR_B32:
return AMDGPU::V_LSHR_B32_e32;
6531 case AMDGPU::S_LSHR_B64:
return AMDGPU::V_LSHR_B64_e64;
6532 case AMDGPU::S_SEXT_I32_I8:
return AMDGPU::V_BFE_I32_e64;
6533 case AMDGPU::S_SEXT_I32_I16:
return AMDGPU::V_BFE_I32_e64;
6534 case AMDGPU::S_BFE_U32:
return AMDGPU::V_BFE_U32_e64;
6535 case AMDGPU::S_BFE_I32:
return AMDGPU::V_BFE_I32_e64;
6536 case AMDGPU::S_BFM_B32:
return AMDGPU::V_BFM_B32_e64;
6537 case AMDGPU::S_BREV_B32:
return AMDGPU::V_BFREV_B32_e32;
6538 case AMDGPU::S_NOT_B32:
return AMDGPU::V_NOT_B32_e32;
6539 case AMDGPU::S_NOT_B64:
return AMDGPU::V_NOT_B32_e32;
6540 case AMDGPU::S_CMP_EQ_I32:
return AMDGPU::V_CMP_EQ_I32_e64;
6541 case AMDGPU::S_CMP_LG_I32:
return AMDGPU::V_CMP_NE_I32_e64;
6542 case AMDGPU::S_CMP_GT_I32:
return AMDGPU::V_CMP_GT_I32_e64;
6543 case AMDGPU::S_CMP_GE_I32:
return AMDGPU::V_CMP_GE_I32_e64;
6544 case AMDGPU::S_CMP_LT_I32:
return AMDGPU::V_CMP_LT_I32_e64;
6545 case AMDGPU::S_CMP_LE_I32:
return AMDGPU::V_CMP_LE_I32_e64;
6546 case AMDGPU::S_CMP_EQ_U32:
return AMDGPU::V_CMP_EQ_U32_e64;
6547 case AMDGPU::S_CMP_LG_U32:
return AMDGPU::V_CMP_NE_U32_e64;
6548 case AMDGPU::S_CMP_GT_U32:
return AMDGPU::V_CMP_GT_U32_e64;
6549 case AMDGPU::S_CMP_GE_U32:
return AMDGPU::V_CMP_GE_U32_e64;
6550 case AMDGPU::S_CMP_LT_U32:
return AMDGPU::V_CMP_LT_U32_e64;
6551 case AMDGPU::S_CMP_LE_U32:
return AMDGPU::V_CMP_LE_U32_e64;
6552 case AMDGPU::S_CMP_EQ_U64:
return AMDGPU::V_CMP_EQ_U64_e64;
6553 case AMDGPU::S_CMP_LG_U64:
return AMDGPU::V_CMP_NE_U64_e64;
6554 case AMDGPU::S_BCNT1_I32_B32:
return AMDGPU::V_BCNT_U32_B32_e64;
6555 case AMDGPU::S_FF1_I32_B32:
return AMDGPU::V_FFBL_B32_e32;
6556 case AMDGPU::S_FLBIT_I32_B32:
return AMDGPU::V_FFBH_U32_e32;
6557 case AMDGPU::S_FLBIT_I32:
return AMDGPU::V_FFBH_I32_e64;
6558 case AMDGPU::S_CBRANCH_SCC0:
return AMDGPU::S_CBRANCH_VCCZ;
6559 case AMDGPU::S_CBRANCH_SCC1:
return AMDGPU::S_CBRANCH_VCCNZ;
6560 case AMDGPU::S_CVT_F32_I32:
return AMDGPU::V_CVT_F32_I32_e64;
6561 case AMDGPU::S_CVT_F32_U32:
return AMDGPU::V_CVT_F32_U32_e64;
6562 case AMDGPU::S_CVT_I32_F32:
return AMDGPU::V_CVT_I32_F32_e64;
6563 case AMDGPU::S_CVT_U32_F32:
return AMDGPU::V_CVT_U32_F32_e64;
6564 case AMDGPU::S_CVT_F32_F16:
6565 case AMDGPU::S_CVT_HI_F32_F16:
6566 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
6567 : AMDGPU::V_CVT_F32_F16_fake16_e64;
6568 case AMDGPU::S_CVT_F16_F32:
6569 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
6570 : AMDGPU::V_CVT_F16_F32_fake16_e64;
6571 case AMDGPU::S_CEIL_F32:
return AMDGPU::V_CEIL_F32_e64;
6572 case AMDGPU::S_FLOOR_F32:
return AMDGPU::V_FLOOR_F32_e64;
6573 case AMDGPU::S_TRUNC_F32:
return AMDGPU::V_TRUNC_F32_e64;
6574 case AMDGPU::S_RNDNE_F32:
return AMDGPU::V_RNDNE_F32_e64;
6575 case AMDGPU::S_CEIL_F16:
6576 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
6577 : AMDGPU::V_CEIL_F16_fake16_e64;
6578 case AMDGPU::S_FLOOR_F16:
6579 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
6580 : AMDGPU::V_FLOOR_F16_fake16_e64;
6581 case AMDGPU::S_TRUNC_F16:
6582 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e64
6583 : AMDGPU::V_TRUNC_F16_fake16_e64;
6584 case AMDGPU::S_RNDNE_F16:
6585 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e64
6586 : AMDGPU::V_RNDNE_F16_fake16_e64;
6587 case AMDGPU::S_ADD_F32:
return AMDGPU::V_ADD_F32_e64;
6588 case AMDGPU::S_SUB_F32:
return AMDGPU::V_SUB_F32_e64;
6589 case AMDGPU::S_MIN_F32:
return AMDGPU::V_MIN_F32_e64;
6590 case AMDGPU::S_MAX_F32:
return AMDGPU::V_MAX_F32_e64;
6591 case AMDGPU::S_MINIMUM_F32:
return AMDGPU::V_MINIMUM_F32_e64;
6592 case AMDGPU::S_MAXIMUM_F32:
return AMDGPU::V_MAXIMUM_F32_e64;
6593 case AMDGPU::S_MUL_F32:
return AMDGPU::V_MUL_F32_e64;
6594 case AMDGPU::S_ADD_F16:
6595 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e64
6596 : AMDGPU::V_ADD_F16_fake16_e64;
6597 case AMDGPU::S_SUB_F16:
6598 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e64
6599 : AMDGPU::V_SUB_F16_fake16_e64;
6600 case AMDGPU::S_MIN_F16:
6601 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e64
6602 : AMDGPU::V_MIN_F16_fake16_e64;
6603 case AMDGPU::S_MAX_F16:
6604 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e64
6605 : AMDGPU::V_MAX_F16_fake16_e64;
6606 case AMDGPU::S_MINIMUM_F16:
6607 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
6608 : AMDGPU::V_MINIMUM_F16_fake16_e64;
6609 case AMDGPU::S_MAXIMUM_F16:
6610 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
6611 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
6612 case AMDGPU::S_MUL_F16:
6613 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e64
6614 : AMDGPU::V_MUL_F16_fake16_e64;
6615 case AMDGPU::S_CVT_PK_RTZ_F16_F32:
return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
6616 case AMDGPU::S_FMAC_F32:
return AMDGPU::V_FMAC_F32_e64;
6617 case AMDGPU::S_FMAC_F16:
6618 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e64
6619 : AMDGPU::V_FMAC_F16_fake16_e64;
6620 case AMDGPU::S_FMAMK_F32:
return AMDGPU::V_FMAMK_F32;
6621 case AMDGPU::S_FMAAK_F32:
return AMDGPU::V_FMAAK_F32;
6622 case AMDGPU::S_CMP_LT_F32:
return AMDGPU::V_CMP_LT_F32_e64;
6623 case AMDGPU::S_CMP_EQ_F32:
return AMDGPU::V_CMP_EQ_F32_e64;
6624 case AMDGPU::S_CMP_LE_F32:
return AMDGPU::V_CMP_LE_F32_e64;
6625 case AMDGPU::S_CMP_GT_F32:
return AMDGPU::V_CMP_GT_F32_e64;
6626 case AMDGPU::S_CMP_LG_F32:
return AMDGPU::V_CMP_LG_F32_e64;
6627 case AMDGPU::S_CMP_GE_F32:
return AMDGPU::V_CMP_GE_F32_e64;
6628 case AMDGPU::S_CMP_O_F32:
return AMDGPU::V_CMP_O_F32_e64;
6629 case AMDGPU::S_CMP_U_F32:
return AMDGPU::V_CMP_U_F32_e64;
6630 case AMDGPU::S_CMP_NGE_F32:
return AMDGPU::V_CMP_NGE_F32_e64;
6631 case AMDGPU::S_CMP_NLG_F32:
return AMDGPU::V_CMP_NLG_F32_e64;
6632 case AMDGPU::S_CMP_NGT_F32:
return AMDGPU::V_CMP_NGT_F32_e64;
6633 case AMDGPU::S_CMP_NLE_F32:
return AMDGPU::V_CMP_NLE_F32_e64;
6634 case AMDGPU::S_CMP_NEQ_F32:
return AMDGPU::V_CMP_NEQ_F32_e64;
6635 case AMDGPU::S_CMP_NLT_F32:
return AMDGPU::V_CMP_NLT_F32_e64;
6636 case AMDGPU::S_CMP_LT_F16:
6637 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
6638 : AMDGPU::V_CMP_LT_F16_fake16_e64;
6639 case AMDGPU::S_CMP_EQ_F16:
6640 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
6641 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
6642 case AMDGPU::S_CMP_LE_F16:
6643 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
6644 : AMDGPU::V_CMP_LE_F16_fake16_e64;
6645 case AMDGPU::S_CMP_GT_F16:
6646 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
6647 : AMDGPU::V_CMP_GT_F16_fake16_e64;
6648 case AMDGPU::S_CMP_LG_F16:
6649 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
6650 : AMDGPU::V_CMP_LG_F16_fake16_e64;
6651 case AMDGPU::S_CMP_GE_F16:
6652 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
6653 : AMDGPU::V_CMP_GE_F16_fake16_e64;
6654 case AMDGPU::S_CMP_O_F16:
6655 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
6656 : AMDGPU::V_CMP_O_F16_fake16_e64;
6657 case AMDGPU::S_CMP_U_F16:
6658 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
6659 : AMDGPU::V_CMP_U_F16_fake16_e64;
6660 case AMDGPU::S_CMP_NGE_F16:
6661 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
6662 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
6663 case AMDGPU::S_CMP_NLG_F16:
6664 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
6665 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
6666 case AMDGPU::S_CMP_NGT_F16:
6667 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
6668 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
6669 case AMDGPU::S_CMP_NLE_F16:
6670 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
6671 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
6672 case AMDGPU::S_CMP_NEQ_F16:
6673 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
6674 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
6675 case AMDGPU::S_CMP_NLT_F16:
6676 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
6677 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
6678 case AMDGPU::V_S_EXP_F32_e64:
return AMDGPU::V_EXP_F32_e64;
6679 case AMDGPU::V_S_EXP_F16_e64:
6680 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e64
6681 : AMDGPU::V_EXP_F16_fake16_e64;
6682 case AMDGPU::V_S_LOG_F32_e64:
return AMDGPU::V_LOG_F32_e64;
6683 case AMDGPU::V_S_LOG_F16_e64:
6684 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e64
6685 : AMDGPU::V_LOG_F16_fake16_e64;
6686 case AMDGPU::V_S_RCP_F32_e64:
return AMDGPU::V_RCP_F32_e64;
6687 case AMDGPU::V_S_RCP_F16_e64:
6688 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e64
6689 : AMDGPU::V_RCP_F16_fake16_e64;
6690 case AMDGPU::V_S_RSQ_F32_e64:
return AMDGPU::V_RSQ_F32_e64;
6691 case AMDGPU::V_S_RSQ_F16_e64:
6692 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e64
6693 : AMDGPU::V_RSQ_F16_fake16_e64;
6694 case AMDGPU::V_S_SQRT_F32_e64:
return AMDGPU::V_SQRT_F32_e64;
6695 case AMDGPU::V_S_SQRT_F16_e64:
6696 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e64
6697 : AMDGPU::V_SQRT_F16_fake16_e64;
6700 "Unexpected scalar opcode without corresponding vector one!");
6749 "Not a whole wave func");
6752 if (
MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||
6753 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
6760 unsigned OpNo)
const {
6762 if (
MI.isVariadic() || OpNo >=
Desc.getNumOperands() ||
6763 Desc.operands()[OpNo].RegClass == -1) {
6766 if (Reg.isVirtual()) {
6768 return MRI.getRegClass(Reg);
6770 return RI.getPhysRegBaseClass(Reg);
6773 int16_t RegClass = getOpRegClassID(
Desc.operands()[OpNo]);
6774 return RegClass < 0 ? nullptr : RI.getRegClass(RegClass);
6779 constexpr AMDGPU::OpName OpNames[] = {
6780 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
6783 int SrcIdx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(), OpNames[
I]);
6784 if (
static_cast<unsigned>(SrcIdx) == OpIdx)
6796 unsigned RCID = getOpRegClassID(
get(
MI.getOpcode()).operands()[OpIdx]);
6798 unsigned Size = RI.getRegSizeInBits(*RC);
6799 unsigned Opcode = (
Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
6800 :
Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
6801 : AMDGPU::V_MOV_B32_e32;
6803 Opcode = AMDGPU::COPY;
6804 else if (RI.isSGPRClass(RC))
6805 Opcode = (
Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
6824 Register Low64 =
MRI.createVirtualRegister(VRC64);
6830 .
addImm(AMDGPU::sub0_sub1)
6832 .
addImm(AMDGPU::sub2_sub3);
6833 }
else if (Opcode == AMDGPU::V_MOV_B16_t16_e64) {
6850 return RI.getSubReg(SuperReg.
getReg(), SubIdx);
6854 Register SubReg =
MRI.createVirtualRegister(SubRC);
6856 unsigned NewSubIdx = RI.composeSubRegIndices(SuperReg.
getSubReg(), SubIdx);
6867 if (SubIdx == AMDGPU::sub0)
6869 if (SubIdx == AMDGPU::sub1)
6881void SIInstrInfo::swapOperands(
MachineInstr &Inst)
const {
6897 if (Reg.isPhysical())
6904 RI.getLargestLegalSuperClass(RC,
MRI.getMF());
6907 return RI.getMatchingSuperRegClass(SuperRC, DRC, MO.
getSubReg()) !=
nullptr;
6910 return RI.getCommonSubClass(DRC, RC) !=
nullptr;
6917 unsigned Opc =
MI.getOpcode();
6930 bool IsAGPR = RI.isAGPR(
MRI, MO.
getReg());
6931 if (IsAGPR && !ST.hasMAIInsts())
6933 if (IsAGPR && (!ST.hasGFX90AInsts() || !
MRI.reservedRegsFrozen()) &&
6937 const int VDstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
6938 const int DataIdx = AMDGPU::getNamedOperandIdx(
6939 Opc,
isDS(
Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
6940 if ((
int)OpIdx == VDstIdx && DataIdx != -1 &&
6941 MI.getOperand(DataIdx).isReg() &&
6942 RI.isAGPR(
MRI,
MI.getOperand(DataIdx).getReg()) != IsAGPR)
6944 if ((
int)OpIdx == DataIdx) {
6945 if (VDstIdx != -1 &&
6946 RI.isAGPR(
MRI,
MI.getOperand(VDstIdx).getReg()) != IsAGPR)
6949 const int Data1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::data1);
6950 if (Data1Idx != -1 &&
MI.getOperand(Data1Idx).isReg() &&
6951 RI.isAGPR(
MRI,
MI.getOperand(Data1Idx).getReg()) != IsAGPR)
6956 if (
Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
6957 (
int)OpIdx == AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0) &&
6961 if (ST.hasFlatScratchHiInB64InstHazard() &&
6968 if (
Opc == AMDGPU::S_BITCMP0_B64 ||
Opc == AMDGPU::S_BITCMP1_B64)
6972 (
int)OpIdx == AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1))
6981 constexpr unsigned NumOps = 3;
6982 constexpr AMDGPU::OpName OpNames[
NumOps * 2] = {
6983 AMDGPU::OpName::src0, AMDGPU::OpName::src1,
6984 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
6985 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
6990 int SrcIdx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(), OpNames[SrcN]);
6993 MO = &
MI.getOperand(SrcIdx);
7000 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), OpNames[
NumOps + SrcN]);
7004 unsigned Mods =
MI.getOperand(ModsIdx).getImm();
7008 return !OpSel && !OpSelHi;
7017 int64_t RegClass = getOpRegClassID(OpInfo);
7019 RegClass != -1 ? RI.getRegClass(RegClass) :
nullptr;
7021 MO = &
MI.getOperand(OpIdx);
7025 if (
isVALU(
MI,
false) && !IsInlineConst &&
7029 int ConstantBusLimit = ST.getConstantBusLimit(
MI.getOpcode());
7030 int LiteralLimit = !
isVOP3(
MI) || ST.hasVOP3Literal() ? 1 : 0;
7034 if (!LiteralLimit--)
7044 for (
unsigned i = 0, e =
MI.getNumOperands(); i != e; ++i) {
7052 if (--ConstantBusLimit <= 0)
7064 if (!LiteralLimit--)
7066 if (--ConstantBusLimit <= 0)
7072 for (
unsigned i = 0, e =
MI.getNumOperands(); i != e; ++i) {
7076 if (!
Op.isReg() && !
Op.isFI() && !
Op.isRegMask() &&
7078 !
Op.isIdenticalTo(*MO))
7100 bool Is64BitOp = Is64BitFPOp ||
7108 (!ST.has64BitLiterals() || InstDesc.
getSize() != 4))
7117 if (!Is64BitFPOp && (int32_t)
Imm < 0 &&
7135 bool IsGFX950Only = ST.hasGFX950Insts();
7136 bool IsGFX940Only = ST.hasGFX940Insts();
7138 if (!IsGFX950Only && !IsGFX940Only)
7156 unsigned Opcode =
MI.getOpcode();
7158 case AMDGPU::V_CVT_PK_BF8_F32_e64:
7159 case AMDGPU::V_CVT_PK_FP8_F32_e64:
7160 case AMDGPU::V_MQSAD_PK_U16_U8_e64:
7161 case AMDGPU::V_MQSAD_U32_U8_e64:
7162 case AMDGPU::V_PK_ADD_F16:
7163 case AMDGPU::V_PK_ADD_F32:
7164 case AMDGPU::V_PK_ADD_I16:
7165 case AMDGPU::V_PK_ADD_U16:
7166 case AMDGPU::V_PK_ASHRREV_I16:
7167 case AMDGPU::V_PK_FMA_F16:
7168 case AMDGPU::V_PK_FMA_F32:
7169 case AMDGPU::V_PK_FMAC_F16_e32:
7170 case AMDGPU::V_PK_FMAC_F16_e64:
7171 case AMDGPU::V_PK_LSHLREV_B16:
7172 case AMDGPU::V_PK_LSHRREV_B16:
7173 case AMDGPU::V_PK_MAD_I16:
7174 case AMDGPU::V_PK_MAD_U16:
7175 case AMDGPU::V_PK_MAX_F16:
7176 case AMDGPU::V_PK_MAX_I16:
7177 case AMDGPU::V_PK_MAX_U16:
7178 case AMDGPU::V_PK_MIN_F16:
7179 case AMDGPU::V_PK_MIN_I16:
7180 case AMDGPU::V_PK_MIN_U16:
7181 case AMDGPU::V_PK_MOV_B32:
7182 case AMDGPU::V_PK_MUL_F16:
7183 case AMDGPU::V_PK_MUL_F32:
7184 case AMDGPU::V_PK_MUL_LO_U16:
7185 case AMDGPU::V_PK_SUB_I16:
7186 case AMDGPU::V_PK_SUB_U16:
7187 case AMDGPU::V_QSAD_PK_U16_U8_e64:
7196 unsigned Opc =
MI.getOpcode();
7199 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
7202 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
7208 if (HasImplicitSGPR && ST.getConstantBusLimit(
Opc) <= 1 && Src0.isReg() &&
7209 RI.isSGPRReg(
MRI, Src0.getReg()))
7215 if (
Opc == AMDGPU::V_WRITELANE_B32) {
7217 if (Src0.isReg() && RI.isVGPR(
MRI, Src0.getReg())) {
7218 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7221 Src0.ChangeToRegister(Reg,
false);
7223 if (Src1.isReg() && RI.isVGPR(
MRI, Src1.getReg())) {
7224 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7228 Src1.ChangeToRegister(Reg,
false);
7234 if (
Opc == AMDGPU::V_FMAC_F32_e32 ||
Opc == AMDGPU::V_FMAC_F16_e32) {
7235 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
7236 if (!RI.isVGPR(
MRI,
MI.getOperand(Src2Idx).getReg()))
7248 if (
Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
7249 RI.isVGPR(
MRI, Src1.getReg())) {
7250 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7254 Src1.ChangeToRegister(Reg,
false);
7262 if (HasImplicitSGPR || !
MI.isCommutable()) {
7272 if ((!Src1.isImm() && !Src1.isReg()) ||
7279 if (CommutedOpc == -1) {
7284 MI.setDesc(
get(CommutedOpc));
7287 unsigned Src0SubReg = Src0.getSubReg();
7288 bool Src0Kill = Src0.isKill();
7291 Src0.ChangeToImmediate(Src1.getImm());
7292 else if (Src1.isReg()) {
7293 Src0.ChangeToRegister(Src1.getReg(),
false,
false, Src1.isKill());
7294 Src0.setSubReg(Src1.getSubReg());
7298 Src1.ChangeToRegister(Src0Reg,
false,
false, Src0Kill);
7299 Src1.setSubReg(Src0SubReg);
7307 unsigned Opc =
MI.getOpcode();
7310 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0),
7311 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1),
7312 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2)
7315 if (
Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
7316 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||
7317 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
7318 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||
7319 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
7320 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
7321 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {
7325 if (Src1.isReg() && !RI.isSGPRClass(
MRI.getRegClass(Src1.getReg()))) {
7326 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7329 Src1.ChangeToRegister(Reg,
false);
7331 if (VOP3Idx[2] != -1) {
7333 if (Src2.isReg() && !RI.isSGPRClass(
MRI.getRegClass(Src2.getReg()))) {
7334 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7337 Src2.ChangeToRegister(Reg,
false);
7343 int ConstantBusLimit = ST.getConstantBusLimit(
Opc);
7344 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
7346 Register SGPRReg = findUsedSGPR(
MI, VOP3Idx);
7348 SGPRsUsed.
insert(SGPRReg);
7352 for (
int Idx : VOP3Idx) {
7361 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
7373 if (!RI.isSGPRClass(RI.getRegClassForReg(
MRI, MO.
getReg())))
7380 if (ConstantBusLimit > 0) {
7392 if ((
Opc == AMDGPU::V_FMAC_F32_e64 ||
Opc == AMDGPU::V_FMAC_F16_e64 ||
7393 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
7394 Opc == AMDGPU::V_FMAC_F16_t16_e64) &&
7395 !RI.isVGPR(
MRI,
MI.getOperand(VOP3Idx[2]).getReg()))
7401 for (
unsigned I = 0;
I < 3; ++
I) {
7414 SRC = RI.getCommonSubClass(SRC, DstRC);
7417 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
7419 if (RI.hasAGPRs(VRC)) {
7420 VRC = RI.getEquivalentVGPRClass(VRC);
7421 Register NewSrcReg =
MRI.createVirtualRegister(VRC);
7423 get(TargetOpcode::COPY), NewSrcReg)
7430 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
7436 for (
unsigned i = 0; i < SubRegs; ++i) {
7437 Register SGPR =
MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
7439 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
7440 .
addReg(SrcReg, {}, RI.getSubRegFromChannel(i));
7446 get(AMDGPU::REG_SEQUENCE), DstReg);
7447 for (
unsigned i = 0; i < SubRegs; ++i) {
7449 MIB.
addImm(RI.getSubRegFromChannel(i));
7462 if (SBase && !RI.isSGPRClass(
MRI.getRegClass(SBase->getReg()))) {
7464 SBase->setReg(SGPR);
7467 if (SOff && !RI.isSGPRReg(
MRI, SOff->
getReg())) {
7475 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::saddr);
7476 if (OldSAddrIdx < 0)
7492 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
7493 if (NewVAddrIdx < 0)
7496 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
7500 if (OldVAddrIdx >= 0) {
7502 VAddrDef =
MRI.getUniqueVRegDef(VAddr.
getReg());
7514 if (OldVAddrIdx == NewVAddrIdx) {
7517 MRI.removeRegOperandFromUseList(&NewVAddr);
7518 MRI.moveOperands(&NewVAddr, &SAddr, 1);
7522 MRI.removeRegOperandFromUseList(&NewVAddr);
7523 MRI.addRegOperandToUseList(&NewVAddr);
7525 assert(OldSAddrIdx == NewVAddrIdx);
7527 if (OldVAddrIdx >= 0) {
7528 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc,
7529 AMDGPU::OpName::vdst_in);
7533 if (NewVDstIn != -1) {
7534 int OldVDstIn = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
7540 if (NewVDstIn != -1) {
7541 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst);
7562 if (!SAddr || RI.isSGPRClass(
MRI.getRegClass(SAddr->
getReg())))
7582 unsigned OpSubReg =
Op.getSubReg();
7585 RI.getRegClassForReg(
MRI, OpReg), OpSubReg);
7591 Register DstReg =
MRI.createVirtualRegister(DstRC);
7592 auto Copy =
BuildMI(InsertMBB,
I,
DL,
get(AMDGPU::COPY), DstReg)
7593 .
addReg(OpReg, {}, OpSubReg);
7595 Op.setSubReg(AMDGPU::NoSubRegister);
7602 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
7605 bool ImpDef = Def->isImplicitDef();
7606 while (!ImpDef && Def && Def->isCopy()) {
7607 if (Def->getOperand(1).getReg().isPhysical())
7609 Def =
MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
7610 ImpDef = Def && Def->isImplicitDef();
7612 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
7628 const auto *BoolXExecRC =
TRI->getWaveMaskRegClass();
7633 bool UseNewExecInstructions =
7642 if (UseNewExecInstructions) {
7677 for (
auto [Idx, ScalarOp] :
enumerate(ScalarOps)) {
7678 unsigned RegSize =
TRI->getRegSizeInBits(ScalarOp->getReg(),
MRI);
7679 unsigned NumSubRegs =
RegSize / 32;
7680 Register VScalarOp = ScalarOp->getReg();
7683 TII.getRegClass(
TII.get(AMDGPU::V_READFIRSTLANE_B32), 1);
7685 if (NumSubRegs == 1) {
7688 TRI->getCommonSubClass(VScalarOpRC, RFLSrcRC);
7689 Common != VScalarOpRC) {
7696 BuildMI(LoopBB,
I,
DL,
TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
7699 if (UseNewExecInstructions) {
7701 TII.get(AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
7704 if (
I == LoopBB.
end())
7709 BuildMI(LoopBB,
I,
DL,
TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
7715 CondReg = NewCondReg;
7727 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7728 ScalarOp->setReg(CurReg);
7731 BuildMI(*ScalarOp->getParent()->getParent(), ScalarOp->getParent(),
DL,
7732 TII.get(AMDGPU::COPY), PhySGPRs[Idx])
7734 ScalarOp->setReg(PhySGPRs[Idx]);
7739 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
7740 "Unhandled register size");
7742 for (
unsigned Idx = 0;
Idx < NumSubRegs;
Idx += 2) {
7749 BuildMI(LoopBB,
I,
DL,
TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
7750 .
addReg(VScalarOp, VScalarOpUndef,
TRI->getSubRegFromChannel(Idx));
7753 BuildMI(LoopBB,
I,
DL,
TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
7754 .
addReg(VScalarOp, VScalarOpUndef,
7755 TRI->getSubRegFromChannel(Idx + 1));
7762 BuildMI(LoopBB,
I,
DL,
TII.get(AMDGPU::REG_SEQUENCE), CurReg)
7769 NumSubRegs <= 2 ? 0 :
TRI->getSubRegFromChannel(Idx, 2);
7771 if (UseNewExecInstructions) {
7773 TII.get(AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
7775 .
addReg(VScalarOp, VScalarOpUndef, SubReg);
7776 if (
I == LoopBB.
end())
7780 BuildMI(LoopBB,
I,
DL,
TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
7782 .
addReg(VScalarOp, VScalarOpUndef, SubReg);
7786 CondReg = NewCondReg;
7798 const auto *SScalarOpRC =
7804 BuildMI(LoopBB,
I,
DL,
TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
7805 unsigned Channel = 0;
7806 for (
Register Piece : ReadlanePieces) {
7807 Merge.addReg(Piece).addImm(
TRI->getSubRegFromChannel(Channel++));
7811 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7812 ScalarOp->setReg(SScalarOp);
7814 BuildMI(*ScalarOp->getParent()->getParent(), ScalarOp->getParent(),
DL,
7815 TII.get(AMDGPU::COPY), PhySGPRs[Idx])
7817 ScalarOp->setReg(PhySGPRs[Idx]);
7825 if (!UseNewExecInstructions) {
7838 if (UseNewExecInstructions) {
7871 assert((PhySGPRs.empty() || PhySGPRs.size() == ScalarOps.
size()) &&
7872 "Physical SGPRs must be empty or match the number of scalar operands");
7878 if (!Begin.isValid())
7880 if (!
End.isValid()) {
7886 const auto *BoolXExecRC =
TRI->getWaveMaskRegClass();
7895 std::numeric_limits<unsigned>::max()) !=
7941 for (
auto &Succ : RemainderBB->
successors()) {
7966static std::tuple<unsigned, unsigned>
7974 TII.buildExtractSubReg(
MI,
MRI, Rsrc, &AMDGPU::VReg_128RegClass,
7975 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
7978 Register Zero64 =
MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
7979 Register SRsrcFormatLo =
MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
7980 Register SRsrcFormatHi =
MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
7981 Register NewSRsrc =
MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
7982 uint64_t RsrcDataFormat =
TII.getDefaultRsrcDataFormat();
7999 .
addImm(AMDGPU::sub0_sub1)
8005 return std::tuple(RsrcPtr, NewSRsrc);
8016 if (ST.useRealTrue16Insts())
8046 if (!LdStIdx->isGPRIdx())
8049 if (!RI.isSGPRReg(
MRI, Idx->getReg()))
8057 if (
MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
8060 if (RI.hasVGPRs(DstRC)) {
8064 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E;
I += 2) {
8066 if (!
Op.isReg() || !
Op.getReg().isVirtual())
8083 if (
MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
8088 if (DstRC != Src0RC) {
8097 if (
MI.getOpcode() == AMDGPU::SI_INIT_M0) {
8099 if (Src.isReg() && RI.hasVectorRegisters(
MRI.getRegClass(Src.getReg())))
8105 if (
MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
8106 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
8107 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
8108 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
8109 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
8110 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
8111 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
8113 if (Src.isReg() && RI.hasVectorRegisters(
MRI.getRegClass(Src.getReg())))
8126 ? AMDGPU::OpName::rsrc
8127 : AMDGPU::OpName::srsrc;
8129 if (SRsrc && !RI.isSGPRClass(
MRI.getRegClass(SRsrc->
getReg())))
8132 AMDGPU::OpName SampOpName =
8133 isMIMG(
MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
8135 if (SSamp && !RI.isSGPRClass(
MRI.getRegClass(SSamp->
getReg())))
8142 if (
MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
8144 if (!RI.isSGPRClass(
MRI.getRegClass(Dest->
getReg()))) {
8150 if (
MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
8152 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8154 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::src0);
8158 Src0.ChangeToRegister(Reg,
false);
8164 if (
MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d2 ||
8165 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d4 ||
8166 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d2 ||
8167 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d4) {
8169 if (Src.isReg() && RI.hasVectorRegisters(
MRI.getRegClass(Src.getReg())))
8176 bool isSoffsetLegal =
true;
8178 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::soffset);
8179 if (SoffsetIdx != -1) {
8182 !RI.isSGPRClass(
MRI.getRegClass(Soffset->
getReg()))) {
8183 isSoffsetLegal =
false;
8187 bool isRsrcLegal =
true;
8189 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::srsrc);
8190 if (RsrcIdx != -1) {
8193 isRsrcLegal =
false;
8197 if (isRsrcLegal && isSoffsetLegal)
8221 Register NewVAddrLo =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8222 Register NewVAddrHi =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8223 Register NewVAddr =
MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8225 const auto *BoolXExecRC = RI.getWaveMaskRegClass();
8226 Register CondReg0 =
MRI.createVirtualRegister(BoolXExecRC);
8227 Register CondReg1 =
MRI.createVirtualRegister(BoolXExecRC);
8229 unsigned RsrcPtr, NewSRsrc;
8236 .
addReg(RsrcPtr, {}, AMDGPU::sub0)
8237 .addReg(VAddr->
getReg(), {}, AMDGPU::sub0)
8243 .
addReg(RsrcPtr, {}, AMDGPU::sub1)
8244 .addReg(VAddr->
getReg(), {}, AMDGPU::sub1)
8257 }
else if (!VAddr && ST.hasAddr64()) {
8261 "FIXME: Need to emit flat atomics here");
8263 unsigned RsrcPtr, NewSRsrc;
8266 Register NewVAddr =
MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8289 MIB.
addImm(CPol->getImm());
8294 MIB.
addImm(TFE->getImm());
8314 MI.removeFromParent();
8319 .
addReg(RsrcPtr, {}, AMDGPU::sub0)
8320 .addImm(AMDGPU::sub0)
8321 .
addReg(RsrcPtr, {}, AMDGPU::sub1)
8322 .addImm(AMDGPU::sub1);
8325 if (!isSoffsetLegal) {
8336 if (!isSoffsetLegal) {
8345 if (InSet.insert(
MI).second)
8349 AMDGPU::getNamedOperandIdx(
MI->getOpcode(), AMDGPU::OpName::srsrc);
8350 if (RsrcIdx != -1) {
8351 DeferredList.insert(
MI);
8356 return DeferredList.contains(
MI);
8366 if (!ST.useRealTrue16Insts())
8369 unsigned Opcode =
MI.getOpcode();
8372 if (OpIdx >=
MI.getNumExplicitOperands() ||
8373 OpIdx >=
get(Opcode).getNumOperands() ||
8374 get(Opcode).operands()[OpIdx].RegClass == -1)
8378 if (!
Op.isReg() || !
Op.getReg().isVirtual() ||
Op.isDef())
8382 if (!RI.isVGPRClass(CurrRC))
8385 int16_t RCID = getOpRegClassID(
get(Opcode).operands()[OpIdx]);
8387 if (RI.getMatchingSuperRegClass(CurrRC, ExpectedRC, AMDGPU::lo16)) {
8389 if (
Op.getSubReg() == AMDGPU::NoSubRegister)
8390 Op.setSubReg(AMDGPU::lo16);
8395 RI.getSubRegisterClass(CurrRC,
Op.getSubReg());
8396 if (RI.getMatchingSuperRegClass(ExpectedRC, CurrSRC, AMDGPU::lo16)) {
8399 Op.setSubReg(AMDGPU::NoSubRegister);
8404 for (
unsigned OpIdx = 0; OpIdx <
MI.getNumExplicitOperands(); OpIdx++)
8411 unsigned SrcSubReg)
const {
8413 Register Dst =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8417 .
addReg(Src, {}, SrcSubReg)
8418 .addImm(AMDGPU::lo16)
8428 assert(
MI->getOpcode() == AMDGPU::SI_CALL_ISEL &&
8429 "This only handle waterfall for SI_CALL_ISEL");
8436 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
8439 while (
End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
8444 while (
End !=
MBB.end() &&
End->isCopy() &&
8445 MI->definesRegister(
End->getOperand(1).getReg(), &RI))
8455 while (!Worklist.
empty()) {
8461 moveToVALUImpl(Worklist, MDT, Inst, WaterFalls, V2SPhyCopiesToErase);
8467 moveToVALUImpl(Worklist, MDT, *Inst, WaterFalls, V2SPhyCopiesToErase);
8469 "Deferred MachineInstr are not supposed to re-populate worklist");
8472 for (
auto &Entry : WaterFalls) {
8473 if (Entry.first->getOpcode() == AMDGPU::SI_CALL_ISEL)
8475 Entry.second.SGPRs);
8478 for (std::pair<MachineInstr *, bool> Entry : V2SPhyCopiesToErase)
8480 Entry.first->eraseFromParent();
8488 if (SubRegIndices.
size() <= 1) {
8489 Register NewDst =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8491 get(AMDGPU::V_READFIRSTLANE_B32), NewDst)
8498 for (int16_t Indice : SubRegIndices) {
8499 Register NewDst =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8501 get(AMDGPU::V_READFIRSTLANE_B32), NewDst)
8508 get(AMDGPU::REG_SEQUENCE), DstReg);
8509 for (
unsigned i = 0; i < SubRegIndices.size(); ++i) {
8511 MIB.
addImm(RI.getSubRegFromChannel(i));
8527 return LdStIdx && !LdStIdx->isGPRIdx();
8529 if (
none_of(Readers, IsMovrelAccess)) {
8530 TII.createReadFirstLaneFromCopyToPhysReg(
MRI, AMDGPU::M0, Copy);
8534 assert(!Copy.getOperand(1).getSubReg() &&
8535 "selection copies a whole register into M0");
8536 Register Idx = Copy.getOperand(1).getReg();
8539 if (IsMovrelAccess(Reader)) {
8541 Reader->findRegisterUseOperand(AMDGPU::M0, &
TII.getRegisterInfo());
8549 FirstLane =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
8550 BuildMI(*Copy.getParent(), Copy, Copy.getDebugLoc(),
8551 TII.get(AMDGPU::V_READFIRSTLANE_B32), FirstLane)
8554 BuildMI(*Reader->getParent(), Reader, Reader->getDebugLoc(),
8555 TII.get(AMDGPU::COPY), AMDGPU::M0)
8571 if (
MI.readsRegister(DstReg, &RI))
8574 if (
MI.modifiesRegister(DstReg, &RI))
8578 if (DstReg == AMDGPU::M0) {
8588 if (
UseMI->getOpcode() == AMDGPU::SI_CALL_ISEL) {
8589 for (
unsigned i = 0; i <
UseMI->getNumOperands(); ++i) {
8590 if (
UseMI->getOperand(i).isReg() &&
8591 UseMI->getOperand(i).getReg() == DstReg) {
8595 V2SCopyInfo.MOs.push_back(MO);
8596 V2SCopyInfo.SGPRs.push_back(DstReg);
8600 }
else if (
UseMI->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG &&
8601 UseMI->getOperand(0).isReg() &&
8602 UseMI->getOperand(0).getReg() == DstReg) {
8607 V2SPhyCopiesToErase[&Inst] =
false;
8629 case AMDGPU::S_ADD_I32:
8630 case AMDGPU::S_SUB_I32: {
8634 std::tie(
Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
8642 case AMDGPU::S_MUL_U64:
8643 if (ST.useVMulU64Inst()) {
8644 NewOpcode = AMDGPU::V_MUL_U64_e64;
8648 splitScalarSMulU64(Worklist, Inst, MDT);
8652 case AMDGPU::S_MUL_U64_U32_PSEUDO:
8653 case AMDGPU::S_MUL_I64_I32_PSEUDO:
8656 splitScalarSMulPseudo(Worklist, Inst, MDT);
8660 case AMDGPU::S_AND_B64:
8661 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
8665 case AMDGPU::S_OR_B64:
8666 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
8670 case AMDGPU::S_XOR_B64:
8671 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
8675 case AMDGPU::S_NAND_B64:
8676 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
8680 case AMDGPU::S_NOR_B64:
8681 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
8685 case AMDGPU::S_XNOR_B64:
8686 if (ST.hasDLInsts())
8687 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
8689 splitScalar64BitXnor(Worklist, Inst, MDT);
8693 case AMDGPU::S_ANDN2_B64:
8694 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
8698 case AMDGPU::S_ORN2_B64:
8699 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
8703 case AMDGPU::S_BREV_B64:
8704 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32,
true);
8708 case AMDGPU::S_NOT_B64:
8709 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
8713 case AMDGPU::S_BCNT1_I32_B64:
8714 splitScalar64BitBCNT(Worklist, Inst);
8718 case AMDGPU::S_BFE_I64:
8719 splitScalar64BitBFE(Worklist, Inst);
8723 case AMDGPU::S_FLBIT_I32_B64:
8724 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32);
8727 case AMDGPU::S_FF1_I32_B64:
8728 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32);
8732 case AMDGPU::S_LSHL_B32:
8733 if (ST.hasOnlyRevVALUShifts()) {
8734 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
8738 case AMDGPU::S_ASHR_I32:
8739 if (ST.hasOnlyRevVALUShifts()) {
8740 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
8744 case AMDGPU::S_LSHR_B32:
8745 if (ST.hasOnlyRevVALUShifts()) {
8746 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
8750 case AMDGPU::S_LSHL_B64:
8751 if (ST.hasOnlyRevVALUShifts()) {
8753 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
8754 : AMDGPU::V_LSHLREV_B64_e64;
8758 case AMDGPU::S_ASHR_I64:
8759 if (ST.hasOnlyRevVALUShifts()) {
8760 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
8764 case AMDGPU::S_LSHR_B64:
8765 if (ST.hasOnlyRevVALUShifts()) {
8766 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
8771 case AMDGPU::S_ABS_I32:
8772 lowerScalarAbs(Worklist, Inst);
8776 case AMDGPU::S_ABSDIFF_I32:
8777 lowerScalarAbsDiff(Worklist, Inst);
8781 case AMDGPU::S_CBRANCH_SCC0:
8782 case AMDGPU::S_CBRANCH_SCC1: {
8785 bool IsSCC = CondReg == AMDGPU::SCC;
8794 case AMDGPU::S_BFE_U64:
8795 case AMDGPU::S_BFM_B64:
8798 case AMDGPU::S_PACK_LL_B32_B16:
8799 case AMDGPU::S_PACK_LH_B32_B16:
8800 case AMDGPU::S_PACK_HL_B32_B16:
8801 case AMDGPU::S_PACK_HH_B32_B16:
8802 movePackToVALU(Worklist,
MRI, Inst);
8806 case AMDGPU::S_XNOR_B32:
8807 lowerScalarXnor(Worklist, Inst);
8811 case AMDGPU::S_NAND_B32:
8812 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
8816 case AMDGPU::S_NOR_B32:
8817 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
8821 case AMDGPU::S_ANDN2_B32:
8822 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
8826 case AMDGPU::S_ORN2_B32:
8827 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
8835 case AMDGPU::S_ADD_CO_PSEUDO:
8836 case AMDGPU::S_SUB_CO_PSEUDO: {
8837 unsigned Opc = (Inst.
getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
8838 ? AMDGPU::V_ADDC_U32_e64
8839 : AMDGPU::V_SUBB_U32_e64;
8840 const auto *CarryRC = RI.getWaveMaskRegClass();
8843 if (!
MRI.constrainRegClass(CarryInReg, CarryRC)) {
8844 Register NewCarryReg =
MRI.createVirtualRegister(CarryRC);
8851 Register DestReg =
MRI.createVirtualRegister(RI.getEquivalentVGPRClass(
8862 addUsersToMoveToVALUWorklist(DestReg,
MRI, Worklist);
8866 case AMDGPU::S_UADDO_PSEUDO:
8867 case AMDGPU::S_USUBO_PSEUDO: {
8873 unsigned Opc = (Inst.
getOpcode() == AMDGPU::S_UADDO_PSEUDO)
8874 ? AMDGPU::V_ADD_CO_U32_e64
8875 : AMDGPU::V_SUB_CO_U32_e64;
8877 RI.getEquivalentVGPRClass(
MRI.getRegClass(Dest0.
getReg()));
8878 Register DestReg =
MRI.createVirtualRegister(NewRC);
8886 MRI.replaceRegWith(Dest0.
getReg(), DestReg);
8887 addUsersToMoveToVALUWorklist(DestReg,
MRI, Worklist);
8891 case AMDGPU::S_LSHL1_ADD_U32:
8892 case AMDGPU::S_LSHL2_ADD_U32:
8893 case AMDGPU::S_LSHL3_ADD_U32:
8894 case AMDGPU::S_LSHL4_ADD_U32: {
8898 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 1
8899 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 2
8900 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 3
8904 RI.getEquivalentVGPRClass(
MRI.getRegClass(Dest.
getReg()));
8905 Register DestReg =
MRI.createVirtualRegister(NewRC);
8913 MRI.replaceRegWith(Dest.
getReg(), DestReg);
8914 addUsersToMoveToVALUWorklist(DestReg,
MRI, Worklist);
8918 case AMDGPU::S_CSELECT_B32:
8919 case AMDGPU::S_CSELECT_B64:
8920 lowerSelect(Worklist, Inst, MDT);
8923 case AMDGPU::S_CMP_EQ_I32:
8924 case AMDGPU::S_CMP_LG_I32:
8925 case AMDGPU::S_CMP_GT_I32:
8926 case AMDGPU::S_CMP_GE_I32:
8927 case AMDGPU::S_CMP_LT_I32:
8928 case AMDGPU::S_CMP_LE_I32:
8929 case AMDGPU::S_CMP_EQ_U32:
8930 case AMDGPU::S_CMP_LG_U32:
8931 case AMDGPU::S_CMP_GT_U32:
8932 case AMDGPU::S_CMP_GE_U32:
8933 case AMDGPU::S_CMP_LT_U32:
8934 case AMDGPU::S_CMP_LE_U32:
8935 case AMDGPU::S_CMP_EQ_U64:
8936 case AMDGPU::S_CMP_LG_U64:
8937 case AMDGPU::S_CMP_LT_F32:
8938 case AMDGPU::S_CMP_EQ_F32:
8939 case AMDGPU::S_CMP_LE_F32:
8940 case AMDGPU::S_CMP_GT_F32:
8941 case AMDGPU::S_CMP_LG_F32:
8942 case AMDGPU::S_CMP_GE_F32:
8943 case AMDGPU::S_CMP_O_F32:
8944 case AMDGPU::S_CMP_U_F32:
8945 case AMDGPU::S_CMP_NGE_F32:
8946 case AMDGPU::S_CMP_NLG_F32:
8947 case AMDGPU::S_CMP_NGT_F32:
8948 case AMDGPU::S_CMP_NLE_F32:
8949 case AMDGPU::S_CMP_NEQ_F32:
8950 case AMDGPU::S_CMP_NLT_F32: {
8951 Register CondReg =
MRI.createVirtualRegister(RI.getWaveMaskRegClass());
8955 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src0_modifiers) >=
8969 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg);
8973 case AMDGPU::S_CMP_LT_F16:
8974 case AMDGPU::S_CMP_EQ_F16:
8975 case AMDGPU::S_CMP_LE_F16:
8976 case AMDGPU::S_CMP_GT_F16:
8977 case AMDGPU::S_CMP_LG_F16:
8978 case AMDGPU::S_CMP_GE_F16:
8979 case AMDGPU::S_CMP_O_F16:
8980 case AMDGPU::S_CMP_U_F16:
8981 case AMDGPU::S_CMP_NGE_F16:
8982 case AMDGPU::S_CMP_NLG_F16:
8983 case AMDGPU::S_CMP_NGT_F16:
8984 case AMDGPU::S_CMP_NLE_F16:
8985 case AMDGPU::S_CMP_NEQ_F16:
8986 case AMDGPU::S_CMP_NLT_F16: {
8987 Register CondReg =
MRI.createVirtualRegister(RI.getWaveMaskRegClass());
9008 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg);
9012 case AMDGPU::S_CVT_HI_F32_F16: {
9013 Register TmpReg =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9014 Register NewDst =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9015 if (ST.useRealTrue16Insts()) {
9020 .
addReg(TmpReg, {}, AMDGPU::hi16)
9036 addUsersToMoveToVALUWorklist(NewDst,
MRI, Worklist);
9040 case AMDGPU::S_MINIMUM_F32:
9041 case AMDGPU::S_MAXIMUM_F32: {
9042 Register NewDst =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9053 addUsersToMoveToVALUWorklist(NewDst,
MRI, Worklist);
9057 case AMDGPU::S_MINIMUM_F16:
9058 case AMDGPU::S_MAXIMUM_F16: {
9059 Register NewDst =
MRI.createVirtualRegister(ST.useRealTrue16Insts()
9060 ? &AMDGPU::VGPR_16RegClass
9061 : &AMDGPU::VGPR_32RegClass);
9072 addUsersToMoveToVALUWorklist(NewDst,
MRI, Worklist);
9076 case AMDGPU::V_S_EXP_F16_e64:
9077 case AMDGPU::V_S_LOG_F16_e64:
9078 case AMDGPU::V_S_RCP_F16_e64:
9079 case AMDGPU::V_S_RSQ_F16_e64:
9080 case AMDGPU::V_S_SQRT_F16_e64: {
9081 Register NewDst =
MRI.createVirtualRegister(ST.useRealTrue16Insts()
9082 ? &AMDGPU::VGPR_16RegClass
9083 : &AMDGPU::VGPR_32RegClass);
9094 addUsersToMoveToVALUWorklist(NewDst,
MRI, Worklist);
9100 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
9108 if (NewOpcode == Opcode) {
9115 V2SPhyCopiesToErase);
9123 RI.getCommonSubClass(NewDstRC, SrcRC)) {
9130 addUsersToMoveToVALUWorklist(DstReg,
MRI, Worklist);
9136 RI.composeSubRegIndices(SrcSubReg, UseMO.getSubReg()));
9137 UseMO.setReg(NewDstReg);
9142 if (!
MRI.constrainRegClass(NewDstReg, CommonRC))
9155 unsigned OpIdx =
UseMI.getOperandNo(&UseMO);
9158 MRI.constrainRegClass(NewDstReg, OpRC);
9168 if (ST.useRealTrue16Insts() && Inst.
isCopy() &&
9172 if (RI.getMatchingSuperRegClass(NewDstRC, SrcRegRC, AMDGPU::lo16)) {
9177 MRI.replaceRegWith(DstReg, NewDstReg);
9178 addUsersToMoveToVALUWorklist(NewDstReg,
MRI, Worklist);
9180 }
else if (RI.getMatchingSuperRegClass(SrcRegRC, NewDstRC,
9183 Register NewDstReg =
MRI.createVirtualRegister(NewDstRC);
9184 MRI.replaceRegWith(DstReg, NewDstReg);
9185 addUsersToMoveToVALUWorklist(NewDstReg,
MRI, Worklist);
9190 Register NewDstReg =
MRI.createVirtualRegister(NewDstRC);
9191 MRI.replaceRegWith(DstReg, NewDstReg);
9193 addUsersToMoveToVALUWorklist(NewDstReg,
MRI, Worklist);
9203 if (AMDGPU::getNamedOperandIdx(NewOpcode,
9204 AMDGPU::OpName::src0_modifiers) >= 0)
9208 NewInstr->addOperand(Src);
9211 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
9214 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
9216 NewInstr.addImm(
Size);
9217 }
else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
9221 }
else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
9226 "Scalar BFE is only implemented for constant width and offset");
9234 if (AMDGPU::getNamedOperandIdx(NewOpcode,
9235 AMDGPU::OpName::src1_modifiers) >= 0)
9237 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0)
9239 if (AMDGPU::getNamedOperandIdx(NewOpcode,
9240 AMDGPU::OpName::src2_modifiers) >= 0)
9242 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0)
9244 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0)
9246 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0)
9248 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0)
9254 NewInstr->addOperand(
Op);
9260 bool DeadSCCDef =
false;
9262 if (
Op.getReg() == AMDGPU::SCC) {
9268 addSCCDefUsersToVALUWorklist(
Op, Inst, Worklist);
9272 addSCCDefsToVALUWorklist(NewInstr, Worklist);
9277 if (NewInstr->getOperand(0).isReg() && NewInstr->getOperand(0).isDef()) {
9278 Register DstReg = NewInstr->getOperand(0).getReg();
9283 NewDstReg =
MRI.createVirtualRegister(NewDstRC);
9284 MRI.replaceRegWith(DstReg, NewDstReg);
9292 NewInstr->findRegisterDefOperand(RI.getVCC(), &RI))
9293 VCCDef->setIsDead();
9299 addUsersToMoveToVALUWorklist(NewDstReg,
MRI, Worklist);
9303std::pair<bool, MachineBasicBlock *>
9306 if (ST.hasAddNoCarryInsts()) {
9315 Register ResultReg =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
9318 assert(
Opc == AMDGPU::S_ADD_I32 ||
Opc == AMDGPU::S_SUB_I32);
9320 unsigned NewOpc =
Opc == AMDGPU::S_ADD_I32 ?
9321 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
9329 MRI.replaceRegWith(OldDstReg, ResultReg);
9332 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
9333 return std::pair(
true, NewBB);
9336 return std::pair(
false,
nullptr);
9353 bool IsSCC = (CondReg == AMDGPU::SCC);
9359 if (!IsSCC &&
Src0.isImm() && (
Src0.getImm() == -1) &&
Src1.isImm() &&
9360 (
Src1.getImm() == 0)) {
9361 for (MachineOperand &UseMO :
9363 MachineInstr &
UseMI = *UseMO.getParent();
9364 switch (
UseMI.getOpcode()) {
9365 case AMDGPU::V_CNDMASK_B16_fake16_e32:
9366 case AMDGPU::V_CNDMASK_B16_fake16_e64:
9367 case AMDGPU::V_CNDMASK_B16_t16_e32:
9368 case AMDGPU::V_CNDMASK_B16_t16_e64:
9369 case AMDGPU::V_CNDMASK_B32_e32:
9370 case AMDGPU::V_CNDMASK_B32_e64:
9371 case AMDGPU::V_CNDMASK_B64_PSEUDO:
9372 if (UseMO.isImplicit() ||
9374 UseMO.setReg(CondReg);
9388 bool CopyFound =
false;
9389 for (MachineInstr &CandI :
9392 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI,
false,
false) !=
9394 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) {
9396 .
addReg(CandI.getOperand(1).getReg());
9408 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
9417 MachineInstr *NewInst;
9418 if (Inst.
getOpcode() == AMDGPU::S_CSELECT_B32) {
9419 NewInst =
BuildMI(
MBB, MII,
DL,
get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg)
9434 addUsersToMoveToVALUWorklist(NewDestReg,
MRI, Worklist);
9449 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9451 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9453 MachineInstrBuilder
Sub =
9463 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
9480 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9482 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9484 MachineInstrBuilder Sub1 =
BuildMI(
MBB, MII,
DL,
get(SubOp), SubResultReg)
9488 MachineInstrBuilder Sub2 =
9501 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
9515 if (ST.hasDLInsts()) {
9525 addUsersToMoveToVALUWorklist(NewDest,
MRI, Worklist);
9531 bool Src0IsSGPR =
Src0.isReg() &&
9533 bool Src1IsSGPR =
Src1.isReg() &&
9547 }
else if (Src1IsSGPR) {
9565 addUsersToMoveToVALUWorklist(NewDest,
MRI, Worklist);
9571 unsigned Opcode)
const {
9595 addUsersToMoveToVALUWorklist(NewDest,
MRI, Worklist);
9600 unsigned Opcode)
const {
9624 addUsersToMoveToVALUWorklist(NewDest,
MRI, Worklist);
9639 const MCInstrDesc &InstDesc =
get(Opcode);
9642 &AMDGPU::SGPR_32RegClass;
9645 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9648 AMDGPU::sub0, Src0SubRC);
9653 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9656 MachineInstr &LoHalf = *
BuildMI(
MBB, MII,
DL, InstDesc, DestSub0).
add(SrcReg0Sub0);
9659 AMDGPU::sub1, Src0SubRC);
9662 MachineInstr &HiHalf = *
BuildMI(
MBB, MII,
DL, InstDesc, DestSub1).
add(SrcReg0Sub1);
9676 Worklist.
insert(&LoHalf);
9677 Worklist.
insert(&HiHalf);
9683 addUsersToMoveToVALUWorklist(FullDestReg,
MRI, Worklist);
9707 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9708 if (RI.isSGPRClass(Src0SubRC))
9709 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);
9711 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9712 if (RI.isSGPRClass(Src1SubRC))
9713 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);
9717 MachineOperand Op0L =
9719 MachineOperand Op1L =
9721 MachineOperand Op0H =
9723 MachineOperand Op1H =
9742 MachineInstr *Op1L_Op0H =
9748 MachineInstr *Op1H_Op0L =
9754 MachineInstr *Carry =
9759 MachineInstr *LoHalf =
9769 MachineInstr *HiHalf =
9792 addUsersToMoveToVALUWorklist(FullDestReg,
MRI, Worklist);
9816 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9817 if (RI.isSGPRClass(Src0SubRC))
9818 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);
9820 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9821 if (RI.isSGPRClass(Src1SubRC))
9822 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);
9826 MachineOperand Op0L =
9828 MachineOperand Op1L =
9832 unsigned NewOpc =
Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
9833 ? AMDGPU::V_MUL_HI_U32_e64
9834 : AMDGPU::V_MUL_HI_I32_e64;
9835 MachineInstr *HiHalf =
9838 MachineInstr *LoHalf =
9857 addUsersToMoveToVALUWorklist(FullDestReg,
MRI, Worklist);
9873 const MCInstrDesc &InstDesc =
get(Opcode);
9876 &AMDGPU::SGPR_32RegClass;
9879 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9882 &AMDGPU::SGPR_32RegClass;
9885 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9888 AMDGPU::sub0, Src0SubRC);
9890 AMDGPU::sub0, Src1SubRC);
9892 AMDGPU::sub1, Src0SubRC);
9894 AMDGPU::sub1, Src1SubRC);
9899 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9902 MachineInstr &LoHalf = *
BuildMI(
MBB, MII,
DL, InstDesc, DestSub0)
9907 MachineInstr &HiHalf = *
BuildMI(
MBB, MII,
DL, InstDesc, DestSub1)
9920 Worklist.
insert(&LoHalf);
9921 Worklist.
insert(&HiHalf);
9924 addUsersToMoveToVALUWorklist(FullDestReg,
MRI, Worklist);
9944 MachineOperand* Op0;
9945 MachineOperand* Op1;
9947 if (
Src0.isReg() && RI.isSGPRReg(
MRI,
Src0.getReg())) {
9980 const MCInstrDesc &InstDesc =
get(AMDGPU::V_BCNT_U32_B32_e64);
9983 &AMDGPU::SGPR_32RegClass;
9989 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9992 AMDGPU::sub0, SrcSubRC);
9994 AMDGPU::sub1, SrcSubRC);
10004 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
10023 Offset == 0 &&
"Not implemented");
10046 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
10056 .
addReg(Src.getReg(), {}, AMDGPU::sub0);
10059 .
addReg(Src.getReg(), {}, AMDGPU::sub0)
10065 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
10084 const MCInstrDesc &InstDesc =
get(Opcode);
10086 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
10089 Src.isReg() ?
MRI.
getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass;
10091 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
10093 MachineOperand SrcRegSub0 =
10095 MachineOperand SrcRegSub1 =
10109 .
addReg(IsCtlz ? MidReg1 : MidReg2);
10113 .
addReg(IsCtlz ? MidReg2 : MidReg1);
10117 addUsersToMoveToVALUWorklist(MidReg4,
MRI, Worklist);
10120void SIInstrInfo::addUsersToMoveToVALUWorklist(
10124 MachineInstr &
UseMI = *MO.getParent();
10128 switch (
UseMI.getOpcode()) {
10131 case AMDGPU::SOFT_WQM:
10132 case AMDGPU::STRICT_WWM:
10133 case AMDGPU::STRICT_WQM:
10134 case AMDGPU::REG_SEQUENCE:
10136 case AMDGPU::INSERT_SUBREG:
10139 OpNo = MO.getOperandNo();
10146 if (!RI.hasVectorRegisters(OpRC))
10163 if (ST.useRealTrue16Insts()) {
10165 if (!
Src0.isReg() || !RI.isVGPR(
MRI,
Src0.getReg())) {
10168 get(
Src0.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), SrcReg0)
10171 SrcReg0 =
Src0.getReg();
10174 if (!
Src1.isReg() || !RI.isVGPR(
MRI,
Src1.getReg())) {
10177 get(
Src1.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), SrcReg1)
10180 SrcReg1 =
Src1.getReg();
10186 auto NewMI =
BuildMI(*
MBB, Inst,
DL,
get(AMDGPU::REG_SEQUENCE), ResultReg);
10188 case AMDGPU::S_PACK_LL_B32_B16:
10190 .addReg(SrcReg0, {},
10191 isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10192 .addImm(AMDGPU::lo16)
10193 .addReg(SrcReg1, {},
10194 isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10195 .addImm(AMDGPU::hi16);
10197 case AMDGPU::S_PACK_LH_B32_B16:
10199 .addReg(SrcReg0, {},
10200 isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10201 .addImm(AMDGPU::lo16)
10202 .addReg(SrcReg1, {}, AMDGPU::hi16)
10203 .addImm(AMDGPU::hi16);
10205 case AMDGPU::S_PACK_HL_B32_B16:
10206 NewMI.addReg(SrcReg0, {}, AMDGPU::hi16)
10207 .addImm(AMDGPU::lo16)
10208 .addReg(SrcReg1, {},
10209 isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10210 .addImm(AMDGPU::hi16);
10212 case AMDGPU::S_PACK_HH_B32_B16:
10213 NewMI.addReg(SrcReg0, {}, AMDGPU::hi16)
10214 .addImm(AMDGPU::lo16)
10215 .addReg(SrcReg1, {}, AMDGPU::hi16)
10216 .addImm(AMDGPU::hi16);
10224 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
10229 case AMDGPU::S_PACK_LL_B32_B16: {
10248 case AMDGPU::S_PACK_LH_B32_B16: {
10258 case AMDGPU::S_PACK_HL_B32_B16: {
10269 case AMDGPU::S_PACK_HH_B32_B16: {
10289 addUsersToMoveToVALUWorklist(ResultReg,
MRI, Worklist);
10298 assert(
Op.isReg() &&
Op.getReg() == AMDGPU::SCC &&
Op.isDef() &&
10299 !
Op.isDead() &&
Op.getParent() == &SCCDefInst);
10300 SmallVector<MachineInstr *, 4> CopyToDelete;
10303 for (MachineInstr &
MI :
10307 int SCCIdx =
MI.findRegisterUseOperandIdx(AMDGPU::SCC, &RI,
false);
10308 if (SCCIdx != -1) {
10311 Register DestReg =
MI.getOperand(0).getReg();
10318 MI.getOperand(SCCIdx).setReg(NewCond);
10324 if (
MI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI,
false,
false) != -1)
10327 for (
auto &Copy : CopyToDelete)
10328 Copy->eraseFromParent();
10336void SIInstrInfo::addSCCDefsToVALUWorklist(
MachineInstr *SCCUseInst,
10342 for (MachineInstr &
MI :
10345 if (
MI.modifiesRegister(AMDGPU::VCC, &RI))
10347 if (
MI.definesRegister(AMDGPU::SCC, &RI)) {
10364 case AMDGPU::REG_SEQUENCE:
10365 case AMDGPU::INSERT_SUBREG:
10367 case AMDGPU::SOFT_WQM:
10368 case AMDGPU::STRICT_WWM:
10369 case AMDGPU::STRICT_WQM: {
10371 if (RI.isAGPRClass(SrcRC)) {
10372 if (RI.isAGPRClass(NewDstRC))
10377 case AMDGPU::REG_SEQUENCE:
10378 case AMDGPU::INSERT_SUBREG:
10379 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);
10382 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
10388 if (!RI.isSGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
10391 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
10405 int OpIndices[3])
const {
10406 const MCInstrDesc &
Desc =
MI.getDesc();
10424 for (
unsigned i = 0; i < 3; ++i) {
10425 int Idx = OpIndices[i];
10429 const MachineOperand &MO =
MI.getOperand(Idx);
10436 RI.getRegClass(getOpRegClassID(
Desc.operands()[Idx]));
10437 bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
10438 if (IsRequiredSGPR)
10444 if (RI.isSGPRClass(RegRC))
10445 UsedSGPRs[i] =
Reg;
10461 if (UsedSGPRs[0]) {
10462 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
10463 SGPRReg = UsedSGPRs[0];
10466 if (!SGPRReg && UsedSGPRs[1]) {
10467 if (UsedSGPRs[1] == UsedSGPRs[2])
10468 SGPRReg = UsedSGPRs[1];
10475 AMDGPU::OpName OperandName)
const {
10476 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)
10479 int Idx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(), OperandName);
10483 return &
MI.getOperand(Idx);
10497 if (ST.isAmdHsaOS()) {
10500 RsrcDataFormat |= (1ULL << 56);
10505 RsrcDataFormat |= (2ULL << 59);
10508 return RsrcDataFormat;
10518 uint64_t EltSizeValue =
Log2_32(ST.getMaxPrivateElementSize(
true)) - 1;
10523 uint64_t IndexStride = ST.isWave64() ? 3 : 2;
10530 Rsrc23 &=
~AMDGPU::RSRC_DATA_FORMAT;
10536 unsigned Opc =
MI.getOpcode();
10542 return get(
Opc).mayLoad() &&
10549 if (!Addr || !Addr->
isFI())
10558 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::vdata);
10560 return MI.getOperand(VDataIdx).getReg();
10570 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::data);
10572 return MI.getOperand(DataIdx).getReg();
10593 if (!
MI.mayStore())
10606 unsigned Opc =
MI.getOpcode();
10608 unsigned DescSize =
Desc.getSize();
10609 if (
Opc == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay)
10610 return DescSize + 12;
10615 unsigned Size = DescSize;
10619 if (
MI.isBranch() && ST.hasOffset3fBug())
10630 bool HasLiteral =
false;
10631 unsigned LiteralSize = 4;
10632 for (
int I = 0, E =
MI.getNumExplicitOperands();
I != E; ++
I) {
10637 if (ST.has64BitLiterals()) {
10638 switch (OpInfo.OperandType) {
10663 return HasLiteral ? DescSize + LiteralSize : DescSize;
10668 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
10672 int RSrcIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::srsrc);
10673 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
10677 case TargetOpcode::BUNDLE:
10678 return getInstBundleSize(
MI);
10679 case TargetOpcode::INLINEASM:
10680 case TargetOpcode::INLINEASM_BR: {
10682 const char *AsmStr =
MI.getOperand(0).getSymbolName();
10686 if (
MI.isMetaInstruction())
10690 const auto *D16Info = AMDGPU::getT16D16Helper(
Opc);
10693 unsigned LoInstOpcode = D16Info->LoOp;
10695 DescSize =
Desc.getSize();
10699 if (
Opc == AMDGPU::V_FMA_MIX_F16_t16 ||
Opc == AMDGPU::V_FMA_MIX_BF16_t16) {
10702 DescSize =
Desc.getSize();
10711 if (
MI.isBranch() && ST.hasOffset3fBug())
10712 return InstSizeVerifyMode::NoVerify;
10713 return InstSizeVerifyMode::ExactSize;
10718 static const std::pair<int, const char *> TargetIndices[] = {
10758std::pair<unsigned, unsigned>
10765 static const std::pair<unsigned, const char *> TargetFlags[] = {
10783 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10801 return AMDGPU::WWM_COPY;
10803 return AMDGPU::COPY;
10820 if (!IsLRSplitInst && Opcode != AMDGPU::IMPLICIT_DEF)
10824 if (RI.isSGPRClass(RI.getRegClassForReg(
MRI, Reg)))
10825 return IsLRSplitInst;
10838 bool IsNullOrVectorRegister =
true;
10842 IsNullOrVectorRegister = !RI.isSGPRClass(RI.getRegClassForReg(
MRI, Reg));
10845 return IsNullOrVectorRegister &&
10847 (!
MI.isTerminator() &&
MI.getOpcode() != AMDGPU::COPY &&
10848 MI.modifiesRegister(AMDGPU::EXEC, &RI)));
10856 if (ST.hasAddNoCarryInsts())
10860 Register UnusedCarry =
MRI.createVirtualRegister(RI.getBoolRC());
10861 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC());
10872 if (ST.hasAddNoCarryInsts())
10876 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC)
10878 : RS.scavengeRegisterBackwards(
10879 *RI.getBoolRC(),
I,
false,
10892 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
10893 case AMDGPU::SI_KILL_I1_TERMINATOR:
10902 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
10903 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
10904 case AMDGPU::SI_KILL_I1_PSEUDO:
10905 return get(AMDGPU::SI_KILL_I1_TERMINATOR);
10917 const unsigned OffsetBits =
10919 return (1 << OffsetBits) - 1;
10923 if (!ST.isWave32())
10926 if (
MI.isInlineAsm())
10929 if (
MI.getNumOperands() <
MI.getDesc().getNumOperands())
10932 for (
auto &
Op :
MI.implicit_operands()) {
10933 if (
Op.isReg() &&
Op.getReg() == AMDGPU::VCC)
10934 Op.setReg(AMDGPU::VCC_LO);
10943 int Idx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::sbase);
10947 const int16_t RCID = getOpRegClassID(
MI.getDesc().operands()[Idx]);
10948 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass);
10964 if (
Imm > MaxImm) {
10965 if (
Imm <= MaxImm + 64) {
10967 Overflow =
Imm - MaxImm;
10982 Overflow =
High - Alignment.value();
10986 if (Overflow > 0) {
10994 if (ST.hasRestrictedSOffset())
10999 SOffset = Overflow;
11037 if (!ST.hasFlatInstOffsets())
11041 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == FlatAddrSpace::FLAT &&
11046 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
11047 FlatVariant == FlatAddrSpace::FlatScratch &&
Offset < 0 &&
11058std::pair<int64_t, int64_t>
11061 int64_t RemainderOffset = COffsetVal;
11062 int64_t ImmField = 0;
11067 if (AllowNegative) {
11069 int64_t
D = 1LL << NumBits;
11070 RemainderOffset = (COffsetVal /
D) *
D;
11071 ImmField = COffsetVal - RemainderOffset;
11073 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
11075 (ImmField % 4) != 0) {
11077 RemainderOffset += ImmField % 4;
11078 ImmField -= ImmField % 4;
11080 }
else if (COffsetVal >= 0) {
11082 RemainderOffset = COffsetVal - ImmField;
11086 assert(RemainderOffset + ImmField == COffsetVal);
11087 return {ImmField, RemainderOffset};
11092 if (ST.hasNegativeScratchOffsetBug() &&
11100 switch (ST.getGeneration()) {
11134 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
11135 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
11136 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
11137 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
11138 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
11139 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
11140 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
11141 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
11148#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \
11149 case OPCODE##_dpp: \
11150 case OPCODE##_e32: \
11164 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
11165 case AMDGPU::V_FMA_F16_gfx9_e64:
11166 case AMDGPU::V_INTERP_P2_F16:
11167 case AMDGPU::V_MAD_F16_e64:
11168 case AMDGPU::V_MAD_U16_e64:
11169 case AMDGPU::V_MAD_I16_e64:
11170 case AMDGPU::V_MAD_U16_gfx9_e64:
11171 case AMDGPU::V_MAD_I16_gfx9_e64:
11180 "SIInsertWaitcnts should have promoted soft waitcnt instructions!");
11188 switch (ST.getGeneration()) {
11201 if (
isMAI(Opcode)) {
11215 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX11_7Insts())
11218 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX1250Insts())
11225 if (ST.hasGFX90AInsts()) {
11226 uint32_t NMCOp = AMDGPU::INSTRUCTION_LIST_END;
11227 if (ST.hasGFX940Insts())
11229 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11231 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11233 if (NMCOp != AMDGPU::INSTRUCTION_LIST_END)
11239 if (MCOp == AMDGPU::INSTRUCTION_LIST_END)
11258 for (
unsigned I = 0, E = (
MI.getNumOperands() - 1)/ 2;
I < E; ++
I)
11259 if (
MI.getOperand(1 + 2 *
I + 1).getImm() == SubReg) {
11260 auto &RegOp =
MI.getOperand(1 + 2 *
I);
11272 switch (
MI.getOpcode()) {
11274 case AMDGPU::REG_SEQUENCE:
11278 case AMDGPU::INSERT_SUBREG:
11279 if (RSR.
SubReg == (
unsigned)
MI.getOperand(3).getImm())
11296 if (!
P.Reg.isVirtual())
11300 auto *DefInst =
MRI.getVRegDef(RSR.Reg);
11301 while (
auto *
MI = DefInst) {
11303 switch (
MI->getOpcode()) {
11305 case AMDGPU::V_MOV_B32_e32: {
11306 auto &Op1 =
MI->getOperand(1);
11311 DefInst =
MRI.getVRegDef(RSR.Reg);
11319 DefInst =
MRI.getVRegDef(RSR.Reg);
11332 assert(
MRI.isSSA() &&
"Must be run on SSA");
11334 auto *
TRI =
MRI.getTargetRegisterInfo();
11335 auto *DefBB =
DefMI.getParent();
11339 if (
UseMI.getParent() != DefBB)
11342 const int MaxInstScan = 20;
11346 auto E =
UseMI.getIterator();
11347 for (
auto I = std::next(
DefMI.getIterator());
I != E; ++
I) {
11348 if (
I->isDebugInstr())
11351 if (++NumInst > MaxInstScan)
11354 if (
I->modifiesRegister(AMDGPU::EXEC,
TRI))
11364 assert(
MRI.isSSA() &&
"Must be run on SSA");
11366 auto *
TRI =
MRI.getTargetRegisterInfo();
11367 auto *DefBB =
DefMI.getParent();
11369 const int MaxUseScan = 10;
11372 for (
auto &
Use :
MRI.use_nodbg_operands(VReg)) {
11373 auto &UseInst = *
Use.getParent();
11376 if (UseInst.getParent() != DefBB || UseInst.isPHI())
11379 if (++NumUse > MaxUseScan)
11386 const int MaxInstScan = 20;
11390 for (
auto I = std::next(
DefMI.getIterator()); ; ++
I) {
11393 if (
I->isDebugInstr())
11396 if (++NumInst > MaxInstScan)
11409 if (Reg == VReg && --NumUse == 0)
11411 }
else if (
TRI->regsOverlap(Reg, AMDGPU::EXEC))
11420 auto Cur =
MBB.begin();
11421 if (Cur !=
MBB.end())
11423 if (!Cur->isPHI() && Cur->readsRegister(Dst,
nullptr))
11426 }
while (Cur !=
MBB.end() && Cur != LastPHIIt);
11435 if (InsPt !=
MBB.end() &&
11436 (InsPt->getOpcode() == AMDGPU::SI_IF ||
11437 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
11438 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
11439 InsPt->definesRegister(Src,
nullptr)) {
11443 .
addReg(Src, {}, SrcSubReg)
11486 if (isFullCopyInstr(
MI)) {
11487 Register DstReg =
MI.getOperand(0).getReg();
11488 Register SrcReg =
MI.getOperand(1).getReg();
11495 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
11499 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass);
11510 unsigned *PredCost)
const {
11511 if (
MI.isBundle()) {
11514 unsigned Lat = 0,
Count = 0;
11515 for (++
I;
I != E &&
I->isBundledWithPred(); ++
I) {
11517 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*
I));
11519 return Lat +
Count - 1;
11522 return SchedModel.computeInstrLatency(&
MI);
11526 if (!ST.hasGFX1250VALUBlockingCycles())
11533 if (
const auto *Entry = AMDGPU::getGFX1250BlockingCyclesInfo(
MI.getOpcode()))
11534 return Entry->GFX1250BlockingCycles;
11542 return *CallAddrOp;
11549 unsigned Opcode =
MI.getOpcode();
11554 LLT DstTy =
MRI.getType(Dst);
11555 LLT SrcTy =
MRI.getType(Src);
11557 unsigned SrcAS = SrcTy.getAddressSpace();
11560 ST.hasGloballyAddressableScratch()
11568 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
11569 return HandleAddrSpaceCast(
MI);
11572 auto IID = GI->getIntrinsicID();
11579 case Intrinsic::amdgcn_if:
11580 case Intrinsic::amdgcn_else:
11589 if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
11599 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||
11600 Opcode == AMDGPU::G_SEXTLOAD) {
11601 if (
MI.memoperands_empty())
11605 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11606 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS ||
11607 mmo->getAddrSpace() == AMDGPUAS::VGPR;
11615 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opcode) ||
11616 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
11617 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
11623 if (Opcode == TargetOpcode::G_DYN_STACKALLOC)
11626 if (Opcode == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
11634 Formatter = std::make_unique<AMDGPUMIRFormatter>(ST);
11635 return Formatter.get();
11643 unsigned opcode =
MI.getOpcode();
11644 if (opcode == AMDGPU::V_READLANE_B32 ||
11645 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
11646 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
11651 if (
MI.isInlineAsm()) {
11657 if (!RC || !RI.isSGPRClass(RC))
11662 if (isCopyInstr(
MI)) {
11666 RI.getPhysRegBaseClass(srcOp.
getReg());
11674 if (
MI.isPreISelOpcode())
11689 if (
MI.memoperands_empty())
11693 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11694 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11709 for (
unsigned I = 0, E =
MI.getNumOperands();
I != E; ++
I) {
11711 if (!
SrcOp.isReg())
11715 if (!Reg || !
SrcOp.readsReg())
11721 if (RegBank && RegBank->
getID() != AMDGPU::SGPRRegBankID)
11748 F,
"ds_ordered_count unsupported for this calling conv"));
11762 Register &SrcReg2, int64_t &CmpMask,
11763 int64_t &CmpValue)
const {
11764 if (!
MI.getOperand(0).isReg() ||
MI.getOperand(0).getSubReg())
11767 switch (
MI.getOpcode()) {
11770 case AMDGPU::S_CMP_EQ_U32:
11771 case AMDGPU::S_CMP_EQ_I32:
11772 case AMDGPU::S_CMP_LG_U32:
11773 case AMDGPU::S_CMP_LG_I32:
11774 case AMDGPU::S_CMP_LT_U32:
11775 case AMDGPU::S_CMP_LT_I32:
11776 case AMDGPU::S_CMP_GT_U32:
11777 case AMDGPU::S_CMP_GT_I32:
11778 case AMDGPU::S_CMP_LE_U32:
11779 case AMDGPU::S_CMP_LE_I32:
11780 case AMDGPU::S_CMP_GE_U32:
11781 case AMDGPU::S_CMP_GE_I32:
11782 case AMDGPU::S_CMP_EQ_U64:
11783 case AMDGPU::S_CMP_LG_U64:
11784 SrcReg =
MI.getOperand(0).getReg();
11785 if (
MI.getOperand(1).isReg()) {
11786 if (
MI.getOperand(1).getSubReg())
11788 SrcReg2 =
MI.getOperand(1).getReg();
11790 }
else if (
MI.getOperand(1).isImm()) {
11792 CmpValue =
MI.getOperand(1).getImm();
11798 case AMDGPU::S_CMPK_EQ_U32:
11799 case AMDGPU::S_CMPK_EQ_I32:
11800 case AMDGPU::S_CMPK_LG_U32:
11801 case AMDGPU::S_CMPK_LG_I32:
11802 case AMDGPU::S_CMPK_LT_U32:
11803 case AMDGPU::S_CMPK_LT_I32:
11804 case AMDGPU::S_CMPK_GT_U32:
11805 case AMDGPU::S_CMPK_GT_I32:
11806 case AMDGPU::S_CMPK_LE_U32:
11807 case AMDGPU::S_CMPK_LE_I32:
11808 case AMDGPU::S_CMPK_GE_U32:
11809 case AMDGPU::S_CMPK_GE_I32:
11810 SrcReg =
MI.getOperand(0).getReg();
11812 CmpValue =
MI.getOperand(1).getImm();
11822 if (S->isLiveIn(AMDGPU::SCC))
11831bool SIInstrInfo::invertSCCUse(
MachineInstr *SCCDef)
const {
11834 bool SCCIsDead =
false;
11837 constexpr unsigned ScanLimit = 12;
11838 unsigned Count = 0;
11841 if (++
Count > ScanLimit)
11843 if (
MI.readsRegister(AMDGPU::SCC, &RI)) {
11844 if (
MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
11845 MI.getOpcode() == AMDGPU::S_CSELECT_B64 ||
11846 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11847 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC1)
11852 if (
MI.definesRegister(AMDGPU::SCC, &RI)) {
11865 for (MachineInstr *
MI : InvertInstr) {
11866 if (
MI->getOpcode() == AMDGPU::S_CSELECT_B32 ||
11867 MI->getOpcode() == AMDGPU::S_CSELECT_B64) {
11869 }
else if (
MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11870 MI->getOpcode() == AMDGPU::S_CBRANCH_SCC1) {
11871 MI->setDesc(
get(
MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0
11872 ? AMDGPU::S_CBRANCH_SCC1
11873 : AMDGPU::S_CBRANCH_SCC0));
11886 bool NeedInversion)
const {
11887 MachineInstr *KillsSCC =
nullptr;
11892 if (
MI.modifiesRegister(AMDGPU::SCC, &RI))
11894 if (
MI.killsRegister(AMDGPU::SCC, &RI))
11897 if (NeedInversion && !invertSCCUse(SCCRedefine))
11899 if (MachineOperand *SccDef =
11901 SccDef->setIsDead(
false);
11910static std::optional<std::pair<int64_t, int64_t>>
11914 if (
Opc != AMDGPU::S_CSELECT_B32 &&
Opc != AMDGPU::S_CSELECT_B64)
11916 std::optional<int64_t>
A =
11920 std::optional<int64_t>
B =
11924 if (
Opc == AMDGPU::S_CSELECT_B32) {
11930 return std::pair(*
A, *
B);
11934 unsigned &NewDefOpc) {
11937 if (Def.getOpcode() != AMDGPU::S_ADD_I32 &&
11938 Def.getOpcode() != AMDGPU::S_ADD_U32)
11944 Def.getMF()->getSubtarget().getInstrInfo());
11946 auto Imm1 =
TII->getImmOrMaterializedImm(
MRI, AddSrc1);
11947 auto Imm2 =
TII->getImmOrMaterializedImm(
MRI, AddSrc2);
11948 if ((!Imm1 || *Imm1 != 1) && (!Imm2 || *Imm2 != 1))
11951 if (Def.getOpcode() == AMDGPU::S_ADD_I32) {
11953 Def.findRegisterDefOperand(AMDGPU::SCC,
nullptr);
11956 NewDefOpc = AMDGPU::S_ADD_U32;
11958 NeedInversion = !NeedInversion;
11963 Register SrcReg2, int64_t CmpMask,
11973 CmpValue = *ImmOpt;
11976 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue,
MRI,
11977 this](
bool NeedInversion) ->
bool {
11982 unsigned NewDefOpc = Def->getOpcode();
11989 auto [
A,
B] = *Consts;
11990 int64_t
C = Def->getOpcode() == AMDGPU::S_CSELECT_B32 ?
Lo_32(CmpValue)
11993 NeedInversion = !NeedInversion;
12005 if (CmpValue != 0 ||
12011 if (!optimizeSCC(Def, &CmpInstr, NeedInversion))
12014 if (NewDefOpc != Def->getOpcode())
12015 Def->setDesc(
get(NewDefOpc));
12024 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&
12025 MRI->use_nodbg_empty(Def->getOperand(0).getReg())) {
12031 if (Def1 && Def1->
getOpcode() == AMDGPU::COPY && Def2 &&
12040 auto [
A,
B] = *Consts;
12041 if (
A == 0 ||
B == 0)
12042 optimizeSCC(
Select, Def,
A == 0);
12051 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue,
MRI,
12052 this](int64_t ExpectedValue,
unsigned SrcSize,
12053 bool IsReversible,
bool IsSigned) ->
bool {
12081 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
12082 Def->getOpcode() != AMDGPU::S_AND_B64)
12086 const auto isMask = [&Mask, SrcSize,
MRI,
12098 SrcOp = &Def->getOperand(2);
12099 else if (isMask(&Def->getOperand(2)))
12100 SrcOp = &Def->getOperand(1);
12108 if (IsSigned && BitNo == SrcSize - 1)
12111 ExpectedValue <<= BitNo;
12113 bool IsReversedCC =
false;
12114 if (CmpValue != ExpectedValue) {
12117 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
12122 Register DefReg = Def->getOperand(0).getReg();
12123 if (IsReversedCC && !
MRI->hasOneNonDBGUse(DefReg))
12126 if (!optimizeSCC(Def, &CmpInstr,
false))
12129 if (!
MRI->use_nodbg_empty(DefReg)) {
12137 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
12138 : AMDGPU::S_BITCMP1_B32
12139 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
12140 : AMDGPU::S_BITCMP1_B64;
12145 Def->eraseFromParent();
12153 case AMDGPU::S_CMP_EQ_U32:
12154 case AMDGPU::S_CMP_EQ_I32:
12155 case AMDGPU::S_CMPK_EQ_U32:
12156 case AMDGPU::S_CMPK_EQ_I32:
12157 return optimizeCmpAnd(1, 32,
true,
false) ||
12158 optimizeCmpSelect(
true);
12159 case AMDGPU::S_CMP_GE_U32:
12160 case AMDGPU::S_CMPK_GE_U32:
12161 return optimizeCmpAnd(1, 32,
false,
false);
12162 case AMDGPU::S_CMP_GE_I32:
12163 case AMDGPU::S_CMPK_GE_I32:
12164 return optimizeCmpAnd(1, 32,
false,
true);
12165 case AMDGPU::S_CMP_EQ_U64:
12166 return optimizeCmpAnd(1, 64,
true,
false) ||
12167 optimizeCmpSelect(
true);
12168 case AMDGPU::S_CMP_LG_U32:
12169 case AMDGPU::S_CMP_LG_I32:
12170 case AMDGPU::S_CMPK_LG_U32:
12171 case AMDGPU::S_CMPK_LG_I32:
12172 return optimizeCmpAnd(0, 32,
true,
false) ||
12173 optimizeCmpSelect(
false);
12174 case AMDGPU::S_CMP_GT_U32:
12175 case AMDGPU::S_CMPK_GT_U32:
12176 return optimizeCmpAnd(0, 32,
false,
false);
12177 case AMDGPU::S_CMP_GT_I32:
12178 case AMDGPU::S_CMPK_GT_I32:
12179 return optimizeCmpAnd(0, 32,
false,
true);
12180 case AMDGPU::S_CMP_LG_U64:
12181 return optimizeCmpAnd(0, 64,
true,
false) ||
12182 optimizeCmpSelect(
false);
12189 AMDGPU::OpName
OpName)
const {
12190 if (!ST.needsAlignedVGPRs())
12193 int OpNo = AMDGPU::getNamedOperandIdx(
MI.getOpcode(),
OpName);
12205 bool IsAGPR = RI.isAGPR(
MRI, DataReg);
12207 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
12210 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
12211 : &AMDGPU::VReg_64_Align2RegClass);
12213 .
addReg(DataReg, {},
Op.getSubReg())
12218 Op.setSubReg(AMDGPU::sub0);
12223 if (!SchedModel.hasInstrSchedModel())
12229 unsigned RepeatRate = 0;
12231 PI = SchedModel.getWriteProcResBegin(SCDesc),
12232 PE = SchedModel.getWriteProcResEnd(SCDesc);
12234 RepeatRate = std::max(RepeatRate, (
unsigned)PI->ReleaseAtCycle);
12251 if (ST.hasGFX1250Insts())
12258 unsigned Opcode =
MI.getOpcode();
12264 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
12265 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
12268 if (!ST.hasGFX940Insts())
12275 constexpr unsigned MinOrderedXDLVAVDSTVGPRs = 8;
12282 return DstRC && RI.isVGPRClass(DstRC) &&
12283 RI.getRegSizeInBits(*DstRC) >= MinOrderedXDLVAVDSTVGPRs * 32;
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Shared AMDGPU asyncmark stage definitions.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
static Register UseReg(const MachineOperand &MO)
const HexagonInstrInfo * TII
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isUndef(const MachineInstr &MI)
TargetInstrInfo::RegSubRegPair RegSubRegPair
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
uint64_t IntrinsicInst * II
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
This file declares the machine register scavenger class.
static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RC, bool Forward)
static std::optional< std::pair< int64_t, int64_t > > getSelectConstants(const SIInstrInfo &TII, const MachineRegisterInfo &MRI, const MachineInstr &Sel)
If Sel is an S_CSELECT* of two different constants A and B, return them, truncated to the width of th...
static void expandLoadStoreIdx(const SIInstrInfo &TII, MachineInstr &MI)
static void lowerVGPRCopyToM0(const SIInstrInfo &TII, MachineInstr &Copy, ArrayRef< MachineInstr * > Readers, MachineRegisterInfo &MRI, MachineDominatorTree *MDT)
static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc)
static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize)
static bool compareMachineOp(const MachineOperand &Op0, const MachineOperand &Op1)
static bool isStride64(unsigned Opc)
static MachineBasicBlock * generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI, ArrayRef< MachineOperand * > ScalarOps, MachineDominatorTree *MDT, MachineBasicBlock::iterator Begin=nullptr, MachineBasicBlock::iterator End=nullptr, ArrayRef< Register > PhySGPRs={})
#define GENERATE_RENAMED_GFX9_CASES(OPCODE)
static std::tuple< unsigned, unsigned > extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc)
static unsigned VOP3OpIdxToSrcN(const MachineInstr &MI, unsigned OpIdx)
static bool followSubRegDef(MachineInstr &MI, TargetInstrInfo::RegSubRegPair &RSR)
static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize)
static MachineInstr * swapImmOperands(MachineInstr &MI, MachineOperand &NonRegOp1, MachineOperand &NonRegOp2)
static void copyFlagsToImplicitVCC(MachineInstr &MI, const MachineOperand &Orig)
static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA, LocationSize WidthB, int OffsetB)
static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, RegScavenger &RS, bool RegsOverlap, Register ImpUseSuperReg=Register())
Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908.
static unsigned getWWMRegSpillSaveOpcode(unsigned Size, bool IsVectorSuperClass)
static bool memOpsHaveSameBaseOperands(ArrayRef< const MachineOperand * > BaseOps1, ArrayRef< const MachineOperand * > BaseOps2)
static unsigned getWWMRegSpillRestoreOpcode(unsigned Size, bool IsVectorSuperClass)
static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI)
static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion, unsigned &NewDefOpc)
static bool isSCCDeadOnExit(MachineBasicBlock *MBB)
static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize)
static unsigned subtargetEncodingFamily(const GCNSubtarget &ST)
static void preserveCondRegFlags(MachineOperand &CondReg, const MachineOperand &OrigCond)
static Register findImplicitSGPRRead(const MachineInstr &MI)
static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc)
static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI)
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef< const MachineOperand * > BaseOps1, const MachineInstr &MI2, ArrayRef< const MachineOperand * > BaseOps2)
static unsigned getSGPRSpillRestoreOpcode(unsigned Size)
static bool isRegOrFI(const MachineOperand &MO)
static bool isVCmp(const SIInstrInfo &TII, const MachineInstr &MI)
Return true if MI is a VALU comparison, i.e.
static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI)
static constexpr AMDGPU::OpName ModifierOpNames[]
static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, const char *Msg="illegal VGPR to SGPR copy")
static MachineInstr * swapRegAndNonRegOperand(MachineInstr &MI, MachineOperand &RegOp, MachineOperand &NonRegOp)
static bool isLegalPkF32OpSel(const SIInstrInfo &TII, const MachineInstr &MI, bool CheckSwappedSrc=false)
static bool shouldReadExec(const MachineInstr &MI)
static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc)
static bool isRenamedInGFX9(int Opcode)
static TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd)
static bool changesVGPRIndexingMode(const MachineInstr &MI)
static bool isSubRegOf(const SIRegisterInfo &TRI, const MachineOperand &SuperVec, const MachineOperand &SubReg)
static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1, AMDGPU::OpName OpName)
Returns true if both nodes have the same value for the given operand Op, or if both nodes do not have...
static std::pair< unsigned, unsigned > splitGlobalAddressRelocFlags(const MachineOperand &SrcOp)
static unsigned getNumOperandsNoGlue(SDNode *Node)
static bool canRemat(const MachineInstr &MI)
static unsigned getAVSpillRestoreOpcode(unsigned Size)
static void emitLoadScalarOpsFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB, MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL, ArrayRef< MachineOperand * > ScalarOps, ArrayRef< Register > PhySGPRs={})
static unsigned getVGPRSpillRestoreOpcode(unsigned Size)
Interface definition for SIInstrInfo.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
const unsigned AndN2WrExecOpc
static const LaneMaskConstants & get(const GCNSubtarget &ST)
const unsigned XorTermOpc
const unsigned MovTermOpc
const unsigned OrSaveExecOpc
const unsigned AndSaveExecOpc
static LLVM_ABI Semantics SemanticsToEnum(const llvm::fltSemantics &Sem)
Class for arbitrary precision integers.
int64_t getSExtValue() const
Get sign extended value.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
Get the first element.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
This class is the base class for the comparison instructions.
uint64_t getZExtValue() const
Opaque handle to a cycle within a GenericCycleInfo that wraps the cycle's preorder index.
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Diagnostic information for unsupported feature in backend.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void getExitingBlocks(CycleRef C, SmallVectorImpl< BlockT * > &TmpStorage) const
Return all blocks of C that have a successor outside of C.
CycleRef getParentCycle(CycleRef C) const
bool contains(CycleRef Outer, CycleRef Inner) const
Returns true iff Outer contains Inner. O(1). Non-strict.
CycleRef getCycle(const BlockT *Block) const
Find the innermost cycle containing Block.
Itinerary data supplied by a subtarget to be used by a target.
constexpr unsigned getAddressSpace() const
This is an important class for using LLVM in a threaded context.
A live range for subregisters.
LiveInterval - This class represents the liveness of a register, or stack slot.
iterator_range< subrange_iterator > subranges()
bool hasInterval(Register Reg) const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveInterval & getInterval(Register Reg)
LLVM_ABI bool shrinkToUses(LiveInterval *li, SmallVectorImpl< MachineInstr * > *dead=nullptr, LaneBitmask SubRangeMask=LaneBitmask::getAll())
After removing some uses of a register, shrink its live range to just the remaining uses.
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
This class represents the liveness of a register, stack slot, etc.
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
static const MCBinaryExpr * createAnd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createAShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
unsigned getSize() const
Return the number of bytes in the encoding of this instruction, or zero if the encoding size cannot b...
ArrayRef< MCPhysReg > implicit_uses() const
Return a list of registers that are potentially read by any instance of this machine instruction.
unsigned getOpcode() const
Return the opcode number for this descriptor.
This holds information about one operand of a machine instruction, indicating the register class for ...
uint8_t OperandType
Information about the type of the operand.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
bool hasSuperClassEq(const MCRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
Wrapper class representing physical registers. Should be passed by value.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx, SMLoc Loc=SMLoc())
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
LLVM_ABI void setVariableValue(const MCExpr *Value)
Helper class for constructing bundles of MachineInstrs.
MachineBasicBlock::instr_iterator begin() const
Return an iterator to the first bundled instruction.
MIBundleBuilder & append(MachineInstr *MI)
Insert MI into MBB by appending it to the instructions in the bundle.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
LLVM_ABI LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI, MCRegister Reg, const_iterator Before, unsigned Neighborhood=10) const
Return whether (physical) register Reg has been defined and not killed as of just before Before.
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
Instructions::const_iterator const_instr_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
@ LQR_Dead
Register is known to be fully dead.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isImmutableObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to an immutable object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
LLVM_ABI void addImplicitDefUseOperands(MachineFunction &MF)
Add all implicit def and use operands to this instruction.
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
mop_range implicit_operands()
bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
LLVM_ABI bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
void untieRegOperand(unsigned OpIdx)
Break any tie involving OpIdx.
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI void eraseFromBundle()
Unlink 'this' from its basic block and delete it.
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
mop_range explicit_operands()
LLVM_ABI void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
bool isMoveImmediate(QueryType Type=IgnoreBundle) const
Return true if this instruction is a move immediate (including conditional moves) instruction.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
filtered_mop_range all_uses()
Returns an iterator range over all operands that are (explicit or implicit) register uses.
LLVM_ABI void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol)
Set a symbol that will be emitted just after the instruction itself.
LLVM_ABI void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo)
Clear all kill flags affecting Reg.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
MachineOperand * findRegisterDefOperand(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
LLVM_ABI unsigned getOperandNo() const
Returns the index of this operand in the instruction that it belongs to.
const GlobalValue * getGlobal() const
LLVM_ABI void ChangeToFrameIndex(int Idx, unsigned TargetFlags=0)
Replace this operand with a frame index.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
LLVM_ABI void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
LLVM_ABI void ChangeToGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
ChangeToGA - Replace this operand with a new global address operand.
void setIsKill(bool Val=true)
LLVM_ABI void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value.
void setOffset(int64_t Offset)
unsigned getTargetFlags() const
static MachineOperand CreateImm(int64_t Val)
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
bool isEarlyClobber() const
Register getReg() const
getReg - Returns the register number.
bool isTargetIndex() const
isTargetIndex - Tests if this is a MO_TargetIndex operand.
void setTargetFlags(unsigned F)
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
@ MO_Immediate
Immediate operand.
@ MO_Register
Register operand.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
int64_t getOffset() const
Return the offset from the symbol in this operand.
bool isFPImm() const
isFPImm - Tests if this is a MO_FPImmediate operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
iterator_range< use_nodbg_iterator > use_nodbg_operands(Register Reg) const
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
const MachineFunction & getMF() const
void setSimpleHint(Register VReg, Register PrefReg)
Specify the preferred (target independent) register allocation hint for the specified virtual registe...
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
iterator_range< use_iterator > use_operands(Register Reg) const
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isLegalMUBUFImmOffset(unsigned Imm) const
bool isInlineConstant(const APInt &Imm) const
void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const
Fix operands in MI to satisfy constant bus requirements.
bool canAddToBBProlog(const MachineInstr &MI) const
static std::optional< unsigned > extractSubregFromGlobalAddressFlags(unsigned TargetFlags, unsigned SubRegIndex)
Return the target flags for a global address operand reading subregister SubRegIndex of a 64-bit glob...
static bool isDS(const MachineInstr &MI)
MachineBasicBlock * legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT=nullptr) const
Legalize all operands in this instruction.
bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0, int64_t &Offset1) const override
unsigned getLiveRangeSplitOpcode(Register Reg, const MachineFunction &MF) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
static bool isNeverUniform(const MachineInstr &MI)
bool isXDLWMMA(const MachineInstr &MI) const
void handleCopyToPhysHelper(SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst, MachineRegisterInfo &MRI, MachineDominatorTree *MDT, DenseMap< MachineInstr *, V2PhysSCopyInfo > &WaterFalls, DenseMap< MachineInstr *, bool > &V2SPhyCopiesToErase) const
bool isBasicBlockPrologue(const MachineInstr &MI, Register Reg=Register()) const override
uint64_t getDefaultRsrcDataFormat() const
static bool isSOPP(const MachineInstr &MI)
bool mayAccessScratch(const MachineInstr &MI) const
bool isIGLP(unsigned Opcode) const
static bool isFLATScratch(const MachineInstr &MI)
Register buildVGPR32FromLo16(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register Src, unsigned SrcSubReg) const
Build a 32-bit VGPR whose low half is the 16-bit value Src.SrcSubReg and whose high half is undefined...
bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, AMDGPU::FlatAddrSpace FlatVariant) const
Returns if Offset is legal for the subtarget as the offset to a FLAT encoded instruction with the giv...
const MCInstrDesc & getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, bool IsSGPR) const
MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DestReg) const
Return a partially built integer add instruction without carry.
bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0, int64_t Offset1, unsigned NumLoads) const override
bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, Align Alignment=Align(4)) const
bool isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const
Replace the instructions opcode with the equivalent VALU opcode.
static bool isSMRD(const MachineInstr &MI)
unsigned getGFX1250BlockingCyclesTable(const MachineInstr &MI) const
GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register Reg, SlotIndexes *Indexes=nullptr) const
void storeRegToStackSlotCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC) const
bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineOperand &MO, const MCOperandInfo &OpInfo) const
Returns true if this operand uses the constant bus.
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
static unsigned getFoldableCopySrcIdx(const MachineInstr &MI)
unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const
Return the size in bytes of the operand OpNo on the given.
void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &LdSt, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const final
static std::optional< int64_t > extractSubregFromImm(int64_t ImmVal, unsigned SubRegIndex)
Return the extracted immediate value in a subregister use from a constant materialized in a super reg...
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isMTBUF(const MachineInstr &MI)
const MCInstrDesc & getIndirectGPRIDXPseudo(unsigned VecSize, bool IsIndirectSrc) const
static bool isDGEMM(unsigned Opcode)
static bool isEXP(const MachineInstr &MI)
static bool isSALU(const MachineInstr &MI)
static bool setsSCCIfResultIsNonZero(const MachineInstr &MI)
const MIRFormatter * getMIRFormatter() const override
static bool isXcntDrain(const MachineInstr &MI)
True if MI implicitly drains XCNT.
void legalizeGenericOperand(MachineBasicBlock &InsertMBB, MachineBasicBlock::iterator I, const TargetRegisterClass *DstRC, MachineOperand &Op, MachineRegisterInfo &MRI, const DebugLoc &DL) const
MachineInstr * buildShrunkInst(MachineInstr &MI, unsigned NewOpcode) const
static bool isVOP2(const MachineInstr &MI)
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
static bool isSDWA(const MachineInstr &MI)
const MCInstrDesc & getKillTerminatorFromPseudo(unsigned Opcode) const
void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned Quantity) const override
static bool isGather4(const MachineInstr &MI)
MachineInstr * getWholeWaveFunctionSetup(MachineFunction &MF) const
static bool isDOT(const MachineInstr &MI)
std::unique_ptr< PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
InstSizeVerifyMode getInstSizeVerifyMode(const MachineInstr &MI) const override
MachineInstr * createPHISourceCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const override
static bool usesTENSOR_CNT(const MachineInstr &MI)
bool hasModifiers(unsigned Opcode) const
Return true if this instruction has any modifiers.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
static bool isSWMMAC(const MachineInstr &MI)
ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *II, const ScheduleDAGMI *DAG) const override
bool isHighLatencyDef(int Opc) const override
void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const
Legalize the OpIndex operand of this instruction by inserting a MOV.
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isVOPC(const MachineInstr &MI)
void removeModOperands(MachineInstr &MI) const
unsigned getRepeatRate(const MachineInstr &MI) const
Get the repeat rate for a VALU instruction from the scheduling model.
unsigned getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC, unsigned Size, const SIMachineFunctionInfo &MFI) const
bool isLegalSingleSGPRReadInstOperand(const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN, const MachineOperand *MO=nullptr) const
Check if MO would be a legal operand for a single-SGPR-read instruction.
bool isXDL(const MachineInstr &MI) const
Register isStackAccess(const MachineInstr &MI, int &FrameIndex, TypeSize &MemBytes) const
static bool isVIMAGE(const MachineInstr &MI)
void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const
static bool isSOP2(const MachineInstr &MI)
static bool isGWS(const MachineInstr &MI)
bool hasRAWDependency(const MachineInstr &FirstMI, const MachineInstr &SecondMI) const
bool isLegalAV64PseudoImm(uint64_t Imm) const
Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
bool isNeverCoissue(MachineInstr &MI) const
static bool isBUF(const MachineInstr &MI)
bool isNonCommutableDPP(const MachineInstr &MI) const
bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const
bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx, unsigned toIdx) const
static bool isFLATGlobal(const MachineInstr &MI)
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isGlobalMemoryObject(const MachineInstr *MI) const override
bool canInsertSelect(const MachineBasicBlock &MBB, ArrayRef< MachineOperand > Cond, Register DstReg, Register TrueReg, Register FalseReg, SelectExpansion &Exp) const override
static bool isVSAMPLE(const MachineInstr &MI)
bool isBufferSMRD(const MachineInstr &MI) const
static bool isKillTerminator(unsigned Opcode)
bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const
If OpX is multicycle, anti-dependencies are not allowed.
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0, unsigned &SrcOpIdx1) const override
void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register Reg, bool IsSCCLive, SlotIndexes *Indexes=nullptr) const
bool hasVALU32BitEncoding(unsigned Opcode) const
Return true if this 64-bit VALU instruction has a 32-bit encoding.
unsigned getBlockingCycles(const MachineInstr &MI) const
unsigned getMovOpcode(const TargetRegisterClass *DstRC) const
Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex, TypeSize &MemBytes) const
unsigned buildExtractSubReg(MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, unsigned SubIdx, const TargetRegisterClass *SubRC) const
void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const
Legalize operands in MI by either commuting it or inserting a copy of src1.
static bool isVPermPk16(unsigned Opcode)
static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA)
bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const final
static bool isTRANS(const MachineInstr &MI)
static bool isImage(const MachineInstr &MI)
static bool isSOPK(const MachineInstr &MI)
const TargetRegisterClass * getOpRegClass(const MachineInstr &MI, unsigned OpNo) const
Return the correct register class for OpNo.
MachineBasicBlock * insertSimulatedTrap(MachineRegisterInfo &MRI, MachineBasicBlock &MBB, MachineInstr &MI, const DebugLoc &DL) const
Build instructions that simulate the behavior of a s_trap 2 instructions for hardware (namely,...
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static unsigned getDSShaderTypeValue(const MachineFunction &MF)
static bool isFoldableCopy(const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
bool expandPostRAPseudo(MachineInstr &MI) const override
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT, ArrayRef< MachineOperand * > ScalarOps, ArrayRef< Register > PhySGPRs={}) const
Wrapper function for generating waterfall for instruction MI This function take into consideration of...
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, Register VReg, unsigned SubReg=0, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
AMDGPU::AsyncStages getAsyncStage(const MachineInstr &MI) const
static bool isSegmentSpecificFLAT(const MachineInstr &MI)
bool isReMaterializableImpl(const MachineInstr &MI) const override
static bool isVOP3(const MCInstrDesc &Desc)
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool physRegUsesConstantBus(const MachineOperand &Reg) const
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const
static bool isDPP(const MachineInstr &MI)
bool analyzeBranchImpl(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const
static bool isMFMA(const MachineInstr &MI)
bool isLowLatencyInstruction(const MachineInstr &MI) const
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is a instruction that moves/copies value from one register to ano...
void mutateAndCleanupImplicit(MachineInstr &MI, const MCInstrDesc &NewDesc) const
ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const
static bool isMAI(const MCInstrDesc &Desc)
static bool isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode)
void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, LaneBitmask UsedLanes=LaneBitmask::getAll()) const override
static bool usesLGKM_CNT(const MachineInstr &MI)
void legalizeOperandsVALUt16(MachineInstr &Inst, MachineRegisterInfo &MRI) const
Fix operands in Inst to fix 16bit SALU to VALU lowering.
bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, const MachineOperand &MO) const
bool canShrink(const MachineInstr &MI, const MachineRegisterInfo &MRI) const
const MachineOperand & getCalleeOperand(const MachineInstr &MI) const override
bool isAsmOnlyOpcode(int MCOp) const
Check if this instruction should only be used by assembler.
bool isAlwaysGDS(uint32_t Opcode) const
static bool isVGPRSpill(const MachineInstr &MI)
ScheduleHazardRecognizer * CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const override
This is used by the post-RA scheduler (SchedulePostRAList.cpp).
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
unsigned getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr &MI, unsigned *PredCost=nullptr) const override
unsigned getVectorRegSpillSaveOpcode(Register Reg, const TargetRegisterClass *RC, unsigned Size, const SIMachineFunctionInfo &MFI, bool NeedsCFI) const
int64_t getNamedImmOperand(const MachineInstr &MI, AMDGPU::OpName OperandName) const
Get required immediate operand.
ArrayRef< std::pair< int, const char * > > getSerializableTargetIndices() const override
bool regUsesConstantBus(const MachineOperand &Reg, const MachineRegisterInfo &MRI) const
static bool isMIMG(const MachineInstr &MI)
MachineOperand buildExtractSubRegOrImm(MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, unsigned SubIdx, const TargetRegisterClass *SubRC) const
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
bool isLegalRegOperand(const MachineRegisterInfo &MRI, const MCOperandInfo &OpInfo, const MachineOperand &MO) const
Check if MO (a register operand) is a legal register for the given operand description or operand ind...
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
bool isVAVDSTOrderedXDL(const MachineInstr &MI) const
On gfx1250, an XDL instruction whose destination spans at least eight VGPRs decrements VA_VDST only a...
unsigned getVALUOp(const MachineInstr &MI) const
static bool modifiesModeRegister(const MachineInstr &MI)
Return true if the instruction modifies the mode register.q.
Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI, const TargetRegisterClass *DstRC=nullptr) const
Copy a value from a VGPR (SrcReg) to SGPR.
bool hasDivergentBranch(const MachineBasicBlock *MBB) const
Return whether the block terminate with divergent branch.
std::pair< int64_t, int64_t > splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, AMDGPU::FlatAddrSpace FlatVariant) const
Split COffsetVal into {immediate offset field, remainder offset} values.
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
void fixImplicitOperands(MachineInstr &MI) const
bool moveFlatAddrToVGPR(MachineInstr &Inst) const
Change SADDR form of a FLAT Inst to its VADDR form if saddr operand was moved to VGPR.
static bool usesASYNC_CNT(const MachineInstr &MI)
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DestReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool isMaskedByExec(Register Reg, const MachineInstr &Use, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Return true if Reg is a lane mask that already has 0 in every bit corresponding to a lane that is ina...
void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const
bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0, AMDGPU::OpName Src0OpName, MachineOperand &Src1, AMDGPU::OpName Src1OpName) const
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
bool isAsync(const MachineInstr &MI) const
bool memOpsHaveSameBase(ArrayRef< const MachineOperand * > BaseOps1, ArrayRef< const MachineOperand * > BaseOps2) const
Return true if the memory operations described by BaseOps1 and BaseOps2 (as returned by getMemOperand...
bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const
This function is used to determine if an instruction can be safely executed under EXEC = 0 without ha...
bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg, int64_t &ImmVal) const override
static bool isAtomic(const MachineInstr &MI)
bool isLiteralOperandLegal(const MCInstrDesc &InstDesc, const MCOperandInfo &OpInfo) const
static bool isWWMRegSpillOpcode(uint32_t Opcode)
static bool sopkIsZext(unsigned Opcode)
static bool isSGPRSpill(const MachineInstr &MI)
static bool isWMMA(const MachineInstr &MI)
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const
Returns true if the instruction could potentially depend on the value of exec.
void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
std::pair< MachineInstr *, MachineInstr * > expandMovDPP64(MachineInstr &MI) const
static bool isSOPC(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
bool isBarrier(unsigned Opcode) const
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx0, unsigned OpIdx1) const override
bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const
int pseudoToMCOpcode(int Opcode) const
Return a target-specific opcode if Opcode is a pseudo instruction.
const MCInstrDesc & getMCOpcodeFromPseudo(unsigned Opcode) const
Return the descriptor of the target-specific machine instruction that corresponds to the specified ps...
static bool usesVM_CNT(const MachineInstr &MI)
MachineInstr * createPHIDestinationCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, Register Dst) const override
static bool isFixedSize(const MachineInstr &MI)
bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo, MachineCycleInfo *CI) const override
LLVM_READONLY int commuteOpcode(unsigned Opc) const
ValueUniformity getValueUniformity(const MachineInstr &MI) const final
uint64_t getScratchRsrcWords23() const
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, AMDGPU::OpName OperandName) const
Returns the operand named Op.
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveIntervals *LIS) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx, const MachineOperand *MO=nullptr) const
Check if MO is a legal operand if it was the OpIdx Operand for MI.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const
Returns true if negative offsets are allowed for the given FlatVariant.
void moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst, DenseMap< MachineInstr *, V2PhysSCopyInfo > &WaterFalls, DenseMap< MachineInstr *, bool > &V2SPhyCopiesToErase) const
static bool isLDSDMA(const MachineInstr &MI)
static bool isVOP1(const MachineInstr &MI)
SIInstrInfo(const GCNSubtarget &ST)
std::optional< int64_t > getImmOrMaterializedImm(const MachineRegisterInfo &MRI, const MachineOperand &Op, MachineInstr **DefMI=nullptr) const
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool hasAnyModifiersSet(const MachineInstr &MI) const
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Register getLongBranchReservedReg() const
bool isWholeWaveFunction() const
Register getStackPtrOffsetReg() const
void setHasSpilledVGPRs(bool Spill=true)
bool isWWMReg(Register Reg) const
bool checkFlag(Register Reg, uint8_t Flag) const
void setHasSpilledSGPRs(bool Spill=true)
unsigned getScratchReservedForDynamicVGPRs() const
ArrayRef< int16_t > getRegSplitParts(const TargetRegisterClass *RC, unsigned EltSize) const
unsigned getHWRegIndex(MCRegister Reg) const
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
bool isSGPRReg(const MachineRegisterInfo &MRI, Register Reg) const
unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const override
unsigned getChannelFromSubReg(unsigned SubReg) const
static bool isSGPRClass(const TargetRegisterClass *RC)
static bool isAGPRClass(const TargetRegisterClass *RC)
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
virtual bool hasVRegLiveness() const
Return true if this DAG supports VReg liveness and RegPressure.
MachineFunction & MF
Machine function.
HazardRecognizer - This determines whether or not an instruction can be issued this cycle,...
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
SlotIndex insertMachineInstrInMaps(MachineInstr &MI, bool Late=false)
Insert the given machine instruction into the mapping.
Implements a dense probed hash-table based set with some number of buckets stored inline.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
virtual ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *, const ScheduleDAGMI *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
virtual MachineInstr * createPHIDestinationCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, Register Dst) const
During PHI eleimination lets target to make necessary checks and insert the copy to the PHI destinati...
virtual const MachineOperand & getCalleeOperand(const MachineInstr &MI) const
Returns the callee operand from the given MI.
virtual void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, LaneBitmask UsedLanes=LaneBitmask::getAll()) const
Re-issue the specified 'original' instruction at the specific location targeting a new destination re...
virtual MachineInstr * createPHISourceCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const
During PHI eleimination lets target to make necessary checks and insert the copy to the PHI destinati...
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool isGlobalMemoryObject(const MachineInstr *MI) const
Returns true if MI is an instruction we are unable to reason about (like a call or something with unm...
virtual bool expandPostRAPseudo(MachineInstr &MI) const
This function is called for all pseudo instructions that remain after register allocation.
const MCAsmInfo & getMCAsmInfo() const
Return target specific asm information.
const MCWriteProcResEntry * ProcResIter
static constexpr TypeSize getFixed(ScalarTy ExactSize)
A Use represents the edge between a Value definition and its users.
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ PRIVATE_ADDRESS
Address space for private memory.
unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst)
const uint64_t RSRC_DATA_FORMAT
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool getWMMAIsXDL(unsigned Opc)
unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc)
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isDPMACCInstruction(unsigned Opc)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_READONLY int32_t getCommuteRev(uint32_t Opcode)
LLVM_READONLY int32_t getCommuteOrig(uint32_t Opcode)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool isGFX12Plus(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
constexpr bool isValidAsyncStageMask(int64_t Mask)
AsyncStages
Bit mask of the async stages tracked by the asyncmark / wait_asyncmark intrinsics.
@ GLOBAL_LOAD_ASYNC_TO_LDS
@ GLOBAL_LOAD_ASYNC_TO_LDS_MCAST
@ GLOBAL_STORE_ASYNC_FROM_LDS
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
LLVM_READONLY int32_t getGlobalVaddrOp(uint32_t Opcode)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
LLVM_READONLY int32_t getMFMAEarlyClobberOp(uint32_t Opcode)
bool getMAIIsGFX940XDL(unsigned Opc)
const uint64_t RSRC_ELEMENT_SIZE_SHIFT
bool isIntrinsicAlwaysUniform(unsigned IntrID)
bool isPackedSingleSGPR64BitInst(unsigned Opc)
The opcode is a packed 64-bit instruction which only reads low 64 bits of a scalar operand and propag...
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
LLVM_READONLY int32_t getIfAddr64Inst(uint32_t Opcode)
Check if Opcode is an Addr64 opcode.
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
const uint64_t RSRC_TID_ENABLE
LLVM_READONLY int32_t getVOPe32(uint32_t Opcode)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
bool isGenericAtomic(unsigned Opc)
LLVM_READNONE bool isInlinableIntLiteral(int64_t Literal)
Is this literal inlinable, and not one of the values intended for floating point values.
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
LLVM_READONLY int32_t getAddr64Inst(uint32_t Opcode)
int32_t getMCOpcode(uint32_t Opcode, unsigned Gen)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_IMM_NOINLINE_FP16
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_INLINE_C_AV64_PSEUDO
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
LLVM_READONLY int32_t getBasicFromSDWAOp(uint32_t Opcode)
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
bool isGFX1250(const MCSubtargetInfo &STI)
bool isSingleSGPRReadInst(unsigned Opc)
Packed instructions that read a single SGPR for SGPR operands, except for 64-bit elements which read ...
bool isExtendedGlobalAddrSpace(unsigned AS)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
const uint64_t RSRC_INDEX_STRIDE_SHIFT
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
LLVM_READONLY int32_t getFlatScratchInstSVfromSS(uint32_t Opcode)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isGraphics(CallingConv::ID CC)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
Not(const Pred &P) -> Not< Pred >
constexpr bool isSDWA(const T &...O)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
LLVM_ABI void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O)
Create RegSubRegPair from a register MachineOperand.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
constexpr uint64_t maxUIntN(uint64_t N)
Gets the maximum value for a N-bit unsigned integer.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI, const MachineInstr &UseMI)
Return false if EXEC is not changed between the def of VReg at DefMI and the use at UseMI.
RegState
Flags to represent properties of register accesses.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ Define
Register definition.
@ NoFlags
No Specific Flags.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
constexpr RegState getKillRegState(bool B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
static const MachineMemOperand::Flags MOCFSB1
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg)
Return the SubReg component from REG_SEQUENCE.
static const MachineMemOperand::Flags MONoClobber
Mark the MMO of a uniform load if there are no potentially clobbering stores on any path from the sta...
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
auto reverse(ContainerTy &&C)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
MachineInstr * getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, const MachineRegisterInfo &MRI)
Return the defining instruction for a given reg:subreg pair skipping copy like instructions and subre...
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI VirtRegInfo AnalyzeVirtRegInBundle(MachineInstr &MI, Register Reg, SmallVectorImpl< std::pair< MachineInstr *, unsigned > > *Ops=nullptr)
AnalyzeVirtRegInBundle - Analyze how the current instruction or bundle uses a virtual register.
static const MachineMemOperand::Flags MOCooperative
Mark the MMO of cooperative load/store atomics.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Xor
Bitwise or logical XOR of integers.
@ Sub
Subtraction of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
bool isTargetSpecificOpcode(unsigned Opcode)
Check whether the given Opcode is a target-specific opcode.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned DefaultMemoryClusterDWordsLimit
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
static const MachineMemOperand::Flags MOLastUse
Mark the MMO of a load as the last use.
constexpr T reverseBits(T Val)
Reverse the bits in Val.
static const MachineMemOperand::Flags MOCFSB0
Use two MMO flags to encode the 2-bit cache fill size field for GFX13+.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
@ End
Run the machine verifier once, at the end of the pipeline.
constexpr RegState getUndefRegState(bool B)
ValueUniformity
Enum describing how values behave with respect to uniformity and divergence, to answer the question: ...
@ AlwaysUniform
The result value is always uniform.
@ NeverUniform
The result value can never be assumed to be uniform.
@ Default
The result value is uniform if and only if all operands are uniform.
static const MachineMemOperand::Flags MOThreadPrivate
Mark the MMO of accesses to memory locations that are never written to by other threads.
bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI)
Return false if EXEC is not changed between the def of VReg at DefMI and all its uses.
MCRegisterClass TargetRegisterClass
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Helper struct for the implementation of 3-address conversion to communicate updates made to instructi...
MachineInstr * RemoveMIUse
Other instruction whose def is no longer used by the converted instruction.
uint8_t GFX1250BlockingCycles
static constexpr uint64_t encode(Fields... Values)
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr bool all() const
This represents a simple continuous liveness interval for a value.
Summarize the scheduling resources required for an instruction of a particular scheduling class.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Utility to store machine instructions worklist.
MachineInstr * top() const
bool isDeferred(MachineInstr *MI)
SetVector< MachineInstr * > & getDeferredList()
void insert(MachineInstr *MI)
A pair composed of a register and a sub-register index.
VirtRegInfo - Information about a virtual register used by a set of operands.
bool Reads
Reads - One of the operands read the virtual register.
bool Writes
Writes - One of the operands writes the virtual register.