29#define GET_REGINFO_TARGET_DESC
30#include "AMDGPUGenRegisterInfo.inc"
33 "amdgpu-spill-sgpr-to-vgpr",
34 cl::desc(
"Enable spilling SGPRs to VGPRs"),
38std::array<std::vector<int16_t>, 16> SIRegisterInfo::RegSplitParts;
39std::array<std::array<uint16_t, 32>, 9> SIRegisterInfo::SubRegFromChannelTable;
46 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 9};
49 const Twine &ErrMsg) {
122 MI->getOperand(0).isKill(),
Index,
RS) {}
137 MovOpc = AMDGPU::S_MOV_B32;
138 NotOpc = AMDGPU::S_NOT_B32;
141 MovOpc = AMDGPU::S_MOV_B64;
142 NotOpc = AMDGPU::S_NOT_B64;
147 SuperReg != AMDGPU::EXEC &&
"exec should never spill");
178 assert(
RS &&
"Cannot spill SGPR to memory without RegScavenger");
207 IsWave32 ? AMDGPU::SGPR_32RegClass : AMDGPU::SGPR_64RegClass;
229 "unhandled SGPR spill to memory");
239 I->getOperand(2).setIsDead();
274 I->getOperand(2).setIsDead();
305 "unhandled SGPR spill to memory");
330 ST.getAMDGPUDwarfFlavour(),
334 assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 &&
335 getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) &&
336 (getSubRegIndexLaneMask(AMDGPU::lo16) |
337 getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() ==
338 getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() &&
339 "getNumCoveredRegs() will not work with generated subreg masks!");
341 RegPressureIgnoredUnits.
resize(getNumRegUnits());
343 for (
auto Reg : AMDGPU::VGPR_16RegClass) {
345 RegPressureIgnoredUnits.
set(*regunits(Reg).begin());
351 static auto InitializeRegSplitPartsOnce = [
this]() {
352 for (
unsigned Idx = 1, E = getNumSubRegIndices() - 1;
Idx < E; ++
Idx) {
353 unsigned Size = getSubRegIdxSize(
Idx);
356 std::vector<int16_t> &Vec = RegSplitParts[
Size / 32 - 1];
357 unsigned Pos = getSubRegIdxOffset(
Idx);
362 unsigned MaxNumParts = 1024 /
Size;
363 Vec.resize(MaxNumParts);
371 static auto InitializeSubRegFromChannelTableOnce = [
this]() {
372 for (
auto &Row : SubRegFromChannelTable)
373 Row.fill(AMDGPU::NoSubRegister);
374 for (
unsigned Idx = 1;
Idx < getNumSubRegIndices(); ++
Idx) {
375 unsigned Width = getSubRegIdxSize(
Idx) / 32;
376 unsigned Offset = getSubRegIdxOffset(
Idx) / 32;
381 unsigned TableIdx = Width - 1;
382 assert(TableIdx < SubRegFromChannelTable.size());
384 SubRegFromChannelTable[TableIdx][
Offset] =
Idx;
388 llvm::call_once(InitializeRegSplitPartsFlag, InitializeRegSplitPartsOnce);
390 InitializeSubRegFromChannelTableOnce);
408 : CSR_AMDGPU_SaveList;
410 return ST.
hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_SaveList
411 : CSR_AMDGPU_SI_Gfx_SaveList;
413 return CSR_AMDGPU_CS_ChainPreserve_SaveList;
416 static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister;
417 return &NoCalleeSavedReg;
434 : CSR_AMDGPU_RegMask;
436 return ST.
hasGFX90AInsts() ? CSR_AMDGPU_SI_Gfx_GFX90AInsts_RegMask
437 : CSR_AMDGPU_SI_Gfx_RegMask;
442 return AMDGPU_AllVGPRs_RegMask;
449 return CSR_AMDGPU_NoRegs_RegMask;
453 return VGPR >= AMDGPU::VGPR0 && VGPR < AMDGPU::VGPR8;
464 if (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::AGPR_32RegClass)
465 return &AMDGPU::AV_32RegClass;
466 if (RC == &AMDGPU::VReg_64RegClass || RC == &AMDGPU::AReg_64RegClass)
467 return &AMDGPU::AV_64RegClass;
468 if (RC == &AMDGPU::VReg_64_Align2RegClass ||
469 RC == &AMDGPU::AReg_64_Align2RegClass)
470 return &AMDGPU::AV_64_Align2RegClass;
471 if (RC == &AMDGPU::VReg_96RegClass || RC == &AMDGPU::AReg_96RegClass)
472 return &AMDGPU::AV_96RegClass;
473 if (RC == &AMDGPU::VReg_96_Align2RegClass ||
474 RC == &AMDGPU::AReg_96_Align2RegClass)
475 return &AMDGPU::AV_96_Align2RegClass;
476 if (RC == &AMDGPU::VReg_128RegClass || RC == &AMDGPU::AReg_128RegClass)
477 return &AMDGPU::AV_128RegClass;
478 if (RC == &AMDGPU::VReg_128_Align2RegClass ||
479 RC == &AMDGPU::AReg_128_Align2RegClass)
480 return &AMDGPU::AV_128_Align2RegClass;
481 if (RC == &AMDGPU::VReg_160RegClass || RC == &AMDGPU::AReg_160RegClass)
482 return &AMDGPU::AV_160RegClass;
483 if (RC == &AMDGPU::VReg_160_Align2RegClass ||
484 RC == &AMDGPU::AReg_160_Align2RegClass)
485 return &AMDGPU::AV_160_Align2RegClass;
486 if (RC == &AMDGPU::VReg_192RegClass || RC == &AMDGPU::AReg_192RegClass)
487 return &AMDGPU::AV_192RegClass;
488 if (RC == &AMDGPU::VReg_192_Align2RegClass ||
489 RC == &AMDGPU::AReg_192_Align2RegClass)
490 return &AMDGPU::AV_192_Align2RegClass;
491 if (RC == &AMDGPU::VReg_256RegClass || RC == &AMDGPU::AReg_256RegClass)
492 return &AMDGPU::AV_256RegClass;
493 if (RC == &AMDGPU::VReg_256_Align2RegClass ||
494 RC == &AMDGPU::AReg_256_Align2RegClass)
495 return &AMDGPU::AV_256_Align2RegClass;
496 if (RC == &AMDGPU::VReg_512RegClass || RC == &AMDGPU::AReg_512RegClass)
497 return &AMDGPU::AV_512RegClass;
498 if (RC == &AMDGPU::VReg_512_Align2RegClass ||
499 RC == &AMDGPU::AReg_512_Align2RegClass)
500 return &AMDGPU::AV_512_Align2RegClass;
501 if (RC == &AMDGPU::VReg_1024RegClass || RC == &AMDGPU::AReg_1024RegClass)
502 return &AMDGPU::AV_1024RegClass;
503 if (RC == &AMDGPU::VReg_1024_Align2RegClass ||
504 RC == &AMDGPU::AReg_1024_Align2RegClass)
505 return &AMDGPU::AV_1024_Align2RegClass;
534 return AMDGPU_AllVGPRs_RegMask;
538 return AMDGPU_AllAGPRs_RegMask;
542 return AMDGPU_AllVectorRegs_RegMask;
546 return AMDGPU_AllAllocatableSRegs_RegMask;
553 assert(NumRegIndex &&
"Not implemented");
554 assert(Channel < SubRegFromChannelTable[NumRegIndex - 1].
size());
555 return SubRegFromChannelTable[NumRegIndex - 1][Channel];
560 const unsigned Align,
563 MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
564 return getMatchingSuperReg(BaseReg, AMDGPU::sub0, RC);
572std::pair<unsigned, unsigned>
576 unsigned MaxNumAGPRs = MaxNumVGPRs;
577 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs();
590 MaxNumAGPRs = MaxNumVGPRs;
592 if (MaxNumVGPRs > TotalNumVGPRs) {
593 MaxNumAGPRs = MaxNumVGPRs - TotalNumVGPRs;
594 MaxNumVGPRs = TotalNumVGPRs;
600 return std::pair(MaxNumVGPRs, MaxNumAGPRs);
613 reserveRegisterTuples(
Reserved, AMDGPU::EXEC);
614 reserveRegisterTuples(
Reserved, AMDGPU::FLAT_SCR);
617 reserveRegisterTuples(
Reserved, AMDGPU::M0);
620 reserveRegisterTuples(
Reserved, AMDGPU::SRC_VCCZ);
621 reserveRegisterTuples(
Reserved, AMDGPU::SRC_EXECZ);
622 reserveRegisterTuples(
Reserved, AMDGPU::SRC_SCC);
625 reserveRegisterTuples(
Reserved, AMDGPU::SRC_SHARED_BASE);
626 reserveRegisterTuples(
Reserved, AMDGPU::SRC_SHARED_LIMIT);
627 reserveRegisterTuples(
Reserved, AMDGPU::SRC_PRIVATE_BASE);
628 reserveRegisterTuples(
Reserved, AMDGPU::SRC_PRIVATE_LIMIT);
631 reserveRegisterTuples(
Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID);
634 reserveRegisterTuples(
Reserved, AMDGPU::XNACK_MASK);
637 reserveRegisterTuples(
Reserved, AMDGPU::LDS_DIRECT);
640 reserveRegisterTuples(
Reserved, AMDGPU::TBA);
641 reserveRegisterTuples(
Reserved, AMDGPU::TMA);
642 reserveRegisterTuples(
Reserved, AMDGPU::TTMP0_TTMP1);
643 reserveRegisterTuples(
Reserved, AMDGPU::TTMP2_TTMP3);
644 reserveRegisterTuples(
Reserved, AMDGPU::TTMP4_TTMP5);
645 reserveRegisterTuples(
Reserved, AMDGPU::TTMP6_TTMP7);
646 reserveRegisterTuples(
Reserved, AMDGPU::TTMP8_TTMP9);
647 reserveRegisterTuples(
Reserved, AMDGPU::TTMP10_TTMP11);
648 reserveRegisterTuples(
Reserved, AMDGPU::TTMP12_TTMP13);
649 reserveRegisterTuples(
Reserved, AMDGPU::TTMP14_TTMP15);
652 reserveRegisterTuples(
Reserved, AMDGPU::SGPR_NULL64);
657 unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
660 unsigned NumRegs =
divideCeil(getRegSizeInBits(*RC), 32);
663 if (Index + NumRegs > MaxNumSGPRs && Index < TotalNumSGPRs)
670 if (ScratchRSrcReg != AMDGPU::NoRegister) {
674 reserveRegisterTuples(
Reserved, ScratchRSrcReg);
678 if (LongBranchReservedReg)
679 reserveRegisterTuples(
Reserved, LongBranchReservedReg);
686 reserveRegisterTuples(
Reserved, StackPtrReg);
687 assert(!isSubRegister(ScratchRSrcReg, StackPtrReg));
692 reserveRegisterTuples(
Reserved, FrameReg);
693 assert(!isSubRegister(ScratchRSrcReg, FrameReg));
698 reserveRegisterTuples(
Reserved, BasePtrReg);
699 assert(!isSubRegister(ScratchRSrcReg, BasePtrReg));
706 reserveRegisterTuples(
Reserved, ExecCopyReg);
714 unsigned NumRegs =
divideCeil(getRegSizeInBits(*RC), 32);
717 if (Index + NumRegs > MaxNumVGPRs)
728 unsigned NumRegs =
divideCeil(getRegSizeInBits(*RC), 32);
731 if (Index + NumRegs > MaxNumAGPRs)
747 if (!NonWWMRegMask.
empty()) {
748 for (
unsigned RegI = AMDGPU::VGPR0, RegE = AMDGPU::VGPR0 + MaxNumVGPRs;
749 RegI < RegE; ++RegI) {
750 if (NonWWMRegMask.
test(RegI))
751 reserveRegisterTuples(
Reserved, RegI);
756 reserveRegisterTuples(
Reserved, Reg);
760 reserveRegisterTuples(
Reserved, Reg);
763 reserveRegisterTuples(
Reserved, Reg);
780 if (
Info->isBottomOfStack())
788 if (
Info->isEntryFunction()) {
822 AMDGPU::OpName::offset);
823 return MI->getOperand(OffIdx).getImm();
828 switch (
MI->getOpcode()) {
829 case AMDGPU::V_ADD_U32_e32:
830 case AMDGPU::V_ADD_U32_e64:
831 case AMDGPU::V_ADD_CO_U32_e32: {
832 int OtherIdx =
Idx == 1 ? 2 : 1;
836 case AMDGPU::V_ADD_CO_U32_e64: {
837 int OtherIdx =
Idx == 2 ? 3 : 2;
849 AMDGPU::OpName::vaddr) ||
851 AMDGPU::OpName::saddr))) &&
852 "Should never see frame index on non-address operand");
864 return Src1.
isImm() || (Src1.
isReg() &&
TRI.isVGPR(
MI.getMF()->getRegInfo(),
869 return Src0.
isImm() || (Src0.
isReg() &&
TRI.isVGPR(
MI.getMF()->getRegInfo(),
878 switch (
MI->getOpcode()) {
879 case AMDGPU::V_ADD_U32_e32: {
887 case AMDGPU::V_ADD_U32_e64:
897 case AMDGPU::V_ADD_CO_U32_e32:
903 return MI->getOperand(3).isDead();
904 case AMDGPU::V_ADD_CO_U32_e64:
906 return MI->getOperand(1).isDead();
918 return !
TII->isLegalMUBUFImmOffset(FullOffset);
931 DL = Ins->getDebugLoc();
937 : AMDGPU::V_MOV_B32_e32;
941 : &AMDGPU::VGPR_32RegClass);
949 Register OffsetReg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
953 : &AMDGPU::VGPR_32RegClass);
969 TII->getAddNoCarry(*
MBB, Ins,
DL, BaseReg)
981 switch (
MI.getOpcode()) {
982 case AMDGPU::V_ADD_U32_e32:
983 case AMDGPU::V_ADD_CO_U32_e32: {
989 if (!ImmOp->
isImm()) {
992 TII->legalizeOperandsVOP2(
MI.getMF()->getRegInfo(),
MI);
997 if (TotalOffset == 0) {
998 MI.setDesc(
TII->get(AMDGPU::COPY));
999 for (
unsigned I =
MI.getNumOperands() - 1;
I != 1; --
I)
1000 MI.removeOperand(
I);
1002 MI.getOperand(1).ChangeToRegister(BaseReg,
false);
1006 ImmOp->
setImm(TotalOffset);
1018 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1021 MI.getOperand(2).ChangeToRegister(BaseRegVGPR,
false);
1023 MI.getOperand(2).ChangeToRegister(BaseReg,
false);
1027 case AMDGPU::V_ADD_U32_e64:
1028 case AMDGPU::V_ADD_CO_U32_e64: {
1029 int Src0Idx =
MI.getNumExplicitDefs();
1035 if (!ImmOp->
isImm()) {
1037 TII->legalizeOperandsVOP3(
MI.getMF()->getRegInfo(),
MI);
1042 if (TotalOffset == 0) {
1043 MI.setDesc(
TII->get(AMDGPU::COPY));
1045 for (
unsigned I =
MI.getNumOperands() - 1;
I != 1; --
I)
1046 MI.removeOperand(
I);
1048 MI.getOperand(1).ChangeToRegister(BaseReg,
false);
1051 ImmOp->
setImm(TotalOffset);
1060 bool IsFlat =
TII->isFLATScratch(
MI);
1064 bool SeenFI =
false;
1076 TII->getNamedOperand(
MI, IsFlat ? AMDGPU::OpName::saddr
1077 : AMDGPU::OpName::vaddr);
1082 assert(FIOp && FIOp->
isFI() &&
"frame index must be address operand");
1088 "offset should be legal");
1090 OffsetOp->
setImm(NewOffset);
1099 assert(
TII->isLegalMUBUFImmOffset(NewOffset) &&
"offset should be legal");
1102 OffsetOp->
setImm(NewOffset);
1109 switch (
MI->getOpcode()) {
1110 case AMDGPU::V_ADD_U32_e32:
1111 case AMDGPU::V_ADD_CO_U32_e32:
1113 case AMDGPU::V_ADD_U32_e64:
1114 case AMDGPU::V_ADD_CO_U32_e64:
1127 return TII->isLegalMUBUFImmOffset(NewOffset);
1138 return &AMDGPU::VGPR_32RegClass;
1145 if (RC == &AMDGPU::SCC_CLASSRegClass)
1154 case AMDGPU::SI_SPILL_S1024_SAVE:
1155 case AMDGPU::SI_SPILL_S1024_RESTORE:
1156 case AMDGPU::SI_SPILL_V1024_SAVE:
1157 case AMDGPU::SI_SPILL_V1024_RESTORE:
1158 case AMDGPU::SI_SPILL_A1024_SAVE:
1159 case AMDGPU::SI_SPILL_A1024_RESTORE:
1160 case AMDGPU::SI_SPILL_AV1024_SAVE:
1161 case AMDGPU::SI_SPILL_AV1024_RESTORE:
1163 case AMDGPU::SI_SPILL_S512_SAVE:
1164 case AMDGPU::SI_SPILL_S512_RESTORE:
1165 case AMDGPU::SI_SPILL_V512_SAVE:
1166 case AMDGPU::SI_SPILL_V512_RESTORE:
1167 case AMDGPU::SI_SPILL_A512_SAVE:
1168 case AMDGPU::SI_SPILL_A512_RESTORE:
1169 case AMDGPU::SI_SPILL_AV512_SAVE:
1170 case AMDGPU::SI_SPILL_AV512_RESTORE:
1172 case AMDGPU::SI_SPILL_S384_SAVE:
1173 case AMDGPU::SI_SPILL_S384_RESTORE:
1174 case AMDGPU::SI_SPILL_V384_SAVE:
1175 case AMDGPU::SI_SPILL_V384_RESTORE:
1176 case AMDGPU::SI_SPILL_A384_SAVE:
1177 case AMDGPU::SI_SPILL_A384_RESTORE:
1178 case AMDGPU::SI_SPILL_AV384_SAVE:
1179 case AMDGPU::SI_SPILL_AV384_RESTORE:
1181 case AMDGPU::SI_SPILL_S352_SAVE:
1182 case AMDGPU::SI_SPILL_S352_RESTORE:
1183 case AMDGPU::SI_SPILL_V352_SAVE:
1184 case AMDGPU::SI_SPILL_V352_RESTORE:
1185 case AMDGPU::SI_SPILL_A352_SAVE:
1186 case AMDGPU::SI_SPILL_A352_RESTORE:
1187 case AMDGPU::SI_SPILL_AV352_SAVE:
1188 case AMDGPU::SI_SPILL_AV352_RESTORE:
1190 case AMDGPU::SI_SPILL_S320_SAVE:
1191 case AMDGPU::SI_SPILL_S320_RESTORE:
1192 case AMDGPU::SI_SPILL_V320_SAVE:
1193 case AMDGPU::SI_SPILL_V320_RESTORE:
1194 case AMDGPU::SI_SPILL_A320_SAVE:
1195 case AMDGPU::SI_SPILL_A320_RESTORE:
1196 case AMDGPU::SI_SPILL_AV320_SAVE:
1197 case AMDGPU::SI_SPILL_AV320_RESTORE:
1199 case AMDGPU::SI_SPILL_S288_SAVE:
1200 case AMDGPU::SI_SPILL_S288_RESTORE:
1201 case AMDGPU::SI_SPILL_V288_SAVE:
1202 case AMDGPU::SI_SPILL_V288_RESTORE:
1203 case AMDGPU::SI_SPILL_A288_SAVE:
1204 case AMDGPU::SI_SPILL_A288_RESTORE:
1205 case AMDGPU::SI_SPILL_AV288_SAVE:
1206 case AMDGPU::SI_SPILL_AV288_RESTORE:
1208 case AMDGPU::SI_SPILL_S256_SAVE:
1209 case AMDGPU::SI_SPILL_S256_RESTORE:
1210 case AMDGPU::SI_SPILL_V256_SAVE:
1211 case AMDGPU::SI_SPILL_V256_RESTORE:
1212 case AMDGPU::SI_SPILL_A256_SAVE:
1213 case AMDGPU::SI_SPILL_A256_RESTORE:
1214 case AMDGPU::SI_SPILL_AV256_SAVE:
1215 case AMDGPU::SI_SPILL_AV256_RESTORE:
1217 case AMDGPU::SI_SPILL_S224_SAVE:
1218 case AMDGPU::SI_SPILL_S224_RESTORE:
1219 case AMDGPU::SI_SPILL_V224_SAVE:
1220 case AMDGPU::SI_SPILL_V224_RESTORE:
1221 case AMDGPU::SI_SPILL_A224_SAVE:
1222 case AMDGPU::SI_SPILL_A224_RESTORE:
1223 case AMDGPU::SI_SPILL_AV224_SAVE:
1224 case AMDGPU::SI_SPILL_AV224_RESTORE:
1226 case AMDGPU::SI_SPILL_S192_SAVE:
1227 case AMDGPU::SI_SPILL_S192_RESTORE:
1228 case AMDGPU::SI_SPILL_V192_SAVE:
1229 case AMDGPU::SI_SPILL_V192_RESTORE:
1230 case AMDGPU::SI_SPILL_A192_SAVE:
1231 case AMDGPU::SI_SPILL_A192_RESTORE:
1232 case AMDGPU::SI_SPILL_AV192_SAVE:
1233 case AMDGPU::SI_SPILL_AV192_RESTORE:
1235 case AMDGPU::SI_SPILL_S160_SAVE:
1236 case AMDGPU::SI_SPILL_S160_RESTORE:
1237 case AMDGPU::SI_SPILL_V160_SAVE:
1238 case AMDGPU::SI_SPILL_V160_RESTORE:
1239 case AMDGPU::SI_SPILL_A160_SAVE:
1240 case AMDGPU::SI_SPILL_A160_RESTORE:
1241 case AMDGPU::SI_SPILL_AV160_SAVE:
1242 case AMDGPU::SI_SPILL_AV160_RESTORE:
1244 case AMDGPU::SI_SPILL_S128_SAVE:
1245 case AMDGPU::SI_SPILL_S128_RESTORE:
1246 case AMDGPU::SI_SPILL_V128_SAVE:
1247 case AMDGPU::SI_SPILL_V128_RESTORE:
1248 case AMDGPU::SI_SPILL_A128_SAVE:
1249 case AMDGPU::SI_SPILL_A128_RESTORE:
1250 case AMDGPU::SI_SPILL_AV128_SAVE:
1251 case AMDGPU::SI_SPILL_AV128_RESTORE:
1253 case AMDGPU::SI_SPILL_S96_SAVE:
1254 case AMDGPU::SI_SPILL_S96_RESTORE:
1255 case AMDGPU::SI_SPILL_V96_SAVE:
1256 case AMDGPU::SI_SPILL_V96_RESTORE:
1257 case AMDGPU::SI_SPILL_A96_SAVE:
1258 case AMDGPU::SI_SPILL_A96_RESTORE:
1259 case AMDGPU::SI_SPILL_AV96_SAVE:
1260 case AMDGPU::SI_SPILL_AV96_RESTORE:
1262 case AMDGPU::SI_SPILL_S64_SAVE:
1263 case AMDGPU::SI_SPILL_S64_RESTORE:
1264 case AMDGPU::SI_SPILL_V64_SAVE:
1265 case AMDGPU::SI_SPILL_V64_RESTORE:
1266 case AMDGPU::SI_SPILL_A64_SAVE:
1267 case AMDGPU::SI_SPILL_A64_RESTORE:
1268 case AMDGPU::SI_SPILL_AV64_SAVE:
1269 case AMDGPU::SI_SPILL_AV64_RESTORE:
1271 case AMDGPU::SI_SPILL_S32_SAVE:
1272 case AMDGPU::SI_SPILL_S32_RESTORE:
1273 case AMDGPU::SI_SPILL_V32_SAVE:
1274 case AMDGPU::SI_SPILL_V32_RESTORE:
1275 case AMDGPU::SI_SPILL_A32_SAVE:
1276 case AMDGPU::SI_SPILL_A32_RESTORE:
1277 case AMDGPU::SI_SPILL_AV32_SAVE:
1278 case AMDGPU::SI_SPILL_AV32_RESTORE:
1279 case AMDGPU::SI_SPILL_WWM_V32_SAVE:
1280 case AMDGPU::SI_SPILL_WWM_V32_RESTORE:
1281 case AMDGPU::SI_SPILL_WWM_AV32_SAVE:
1282 case AMDGPU::SI_SPILL_WWM_AV32_RESTORE:
1290 case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
1291 return AMDGPU::BUFFER_STORE_DWORD_OFFSET;
1292 case AMDGPU::BUFFER_STORE_BYTE_OFFEN:
1293 return AMDGPU::BUFFER_STORE_BYTE_OFFSET;
1294 case AMDGPU::BUFFER_STORE_SHORT_OFFEN:
1295 return AMDGPU::BUFFER_STORE_SHORT_OFFSET;
1296 case AMDGPU::BUFFER_STORE_DWORDX2_OFFEN:
1297 return AMDGPU::BUFFER_STORE_DWORDX2_OFFSET;
1298 case AMDGPU::BUFFER_STORE_DWORDX3_OFFEN:
1299 return AMDGPU::BUFFER_STORE_DWORDX3_OFFSET;
1300 case AMDGPU::BUFFER_STORE_DWORDX4_OFFEN:
1301 return AMDGPU::BUFFER_STORE_DWORDX4_OFFSET;
1302 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN:
1303 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET;
1304 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN:
1305 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET;
1313 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
1314 return AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
1315 case AMDGPU::BUFFER_LOAD_UBYTE_OFFEN:
1316 return AMDGPU::BUFFER_LOAD_UBYTE_OFFSET;
1317 case AMDGPU::BUFFER_LOAD_SBYTE_OFFEN:
1318 return AMDGPU::BUFFER_LOAD_SBYTE_OFFSET;
1319 case AMDGPU::BUFFER_LOAD_USHORT_OFFEN:
1320 return AMDGPU::BUFFER_LOAD_USHORT_OFFSET;
1321 case AMDGPU::BUFFER_LOAD_SSHORT_OFFEN:
1322 return AMDGPU::BUFFER_LOAD_SSHORT_OFFSET;
1323 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN:
1324 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET;
1325 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN:
1326 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET;
1327 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN:
1328 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET;
1329 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN:
1330 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET;
1331 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN:
1332 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET;
1333 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN:
1334 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET;
1335 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN:
1336 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET;
1337 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN:
1338 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET;
1339 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN:
1340 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET;
1348 case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
1349 return AMDGPU::BUFFER_STORE_DWORD_OFFEN;
1350 case AMDGPU::BUFFER_STORE_BYTE_OFFSET:
1351 return AMDGPU::BUFFER_STORE_BYTE_OFFEN;
1352 case AMDGPU::BUFFER_STORE_SHORT_OFFSET:
1353 return AMDGPU::BUFFER_STORE_SHORT_OFFEN;
1354 case AMDGPU::BUFFER_STORE_DWORDX2_OFFSET:
1355 return AMDGPU::BUFFER_STORE_DWORDX2_OFFEN;
1356 case AMDGPU::BUFFER_STORE_DWORDX3_OFFSET:
1357 return AMDGPU::BUFFER_STORE_DWORDX3_OFFEN;
1358 case AMDGPU::BUFFER_STORE_DWORDX4_OFFSET:
1359 return AMDGPU::BUFFER_STORE_DWORDX4_OFFEN;
1360 case AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFSET:
1361 return AMDGPU::BUFFER_STORE_SHORT_D16_HI_OFFEN;
1362 case AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFSET:
1363 return AMDGPU::BUFFER_STORE_BYTE_D16_HI_OFFEN;
1371 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
1372 return AMDGPU::BUFFER_LOAD_DWORD_OFFEN;
1373 case AMDGPU::BUFFER_LOAD_UBYTE_OFFSET:
1374 return AMDGPU::BUFFER_LOAD_UBYTE_OFFEN;
1375 case AMDGPU::BUFFER_LOAD_SBYTE_OFFSET:
1376 return AMDGPU::BUFFER_LOAD_SBYTE_OFFEN;
1377 case AMDGPU::BUFFER_LOAD_USHORT_OFFSET:
1378 return AMDGPU::BUFFER_LOAD_USHORT_OFFEN;
1379 case AMDGPU::BUFFER_LOAD_SSHORT_OFFSET:
1380 return AMDGPU::BUFFER_LOAD_SSHORT_OFFEN;
1381 case AMDGPU::BUFFER_LOAD_DWORDX2_OFFSET:
1382 return AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN;
1383 case AMDGPU::BUFFER_LOAD_DWORDX3_OFFSET:
1384 return AMDGPU::BUFFER_LOAD_DWORDX3_OFFEN;
1385 case AMDGPU::BUFFER_LOAD_DWORDX4_OFFSET:
1386 return AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN;
1387 case AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFSET:
1388 return AMDGPU::BUFFER_LOAD_UBYTE_D16_OFFEN;
1389 case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET:
1390 return AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN;
1391 case AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFSET:
1392 return AMDGPU::BUFFER_LOAD_SBYTE_D16_OFFEN;
1393 case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET:
1394 return AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN;
1395 case AMDGPU::BUFFER_LOAD_SHORT_D16_OFFSET:
1396 return AMDGPU::BUFFER_LOAD_SHORT_D16_OFFEN;
1397 case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET:
1398 return AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN;
1407 int Index,
unsigned Lane,
1408 unsigned ValueReg,
bool IsKill) {
1415 if (Reg == AMDGPU::NoRegister)
1418 bool IsStore =
MI->mayStore();
1422 unsigned Dst = IsStore ? Reg : ValueReg;
1423 unsigned Src = IsStore ? ValueReg : Reg;
1424 bool IsVGPR =
TRI->isVGPR(
MRI, Reg);
1426 if (IsVGPR ==
TRI->isVGPR(
MRI, ValueReg)) {
1436 unsigned Opc = (IsStore ^ IsVGPR) ? AMDGPU::V_ACCVGPR_WRITE_B32_e64
1437 : AMDGPU::V_ACCVGPR_READ_B32_e64;
1455 bool IsStore =
MI->mayStore();
1457 unsigned Opc =
MI->getOpcode();
1458 int LoadStoreOp = IsStore ?
1460 if (LoadStoreOp == -1)
1470 .
add(*
TII->getNamedOperand(*
MI, AMDGPU::OpName::srsrc))
1471 .
add(*
TII->getNamedOperand(*
MI, AMDGPU::OpName::soffset))
1478 AMDGPU::OpName::vdata_in);
1480 NewMI.
add(*VDataIn);
1485 unsigned LoadStoreOp,
1487 bool IsStore =
TII->get(LoadStoreOp).mayStore();
1494 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
1495 : AMDGPU::SCRATCH_LOAD_DWORD_SADDR;
1498 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX2_SADDR
1499 : AMDGPU::SCRATCH_LOAD_DWORDX2_SADDR;
1502 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX3_SADDR
1503 : AMDGPU::SCRATCH_LOAD_DWORDX3_SADDR;
1506 LoadStoreOp = IsStore ? AMDGPU::SCRATCH_STORE_DWORDX4_SADDR
1507 : AMDGPU::SCRATCH_LOAD_DWORDX4_SADDR;
1523 unsigned LoadStoreOp,
int Index,
Register ValueReg,
bool IsKill,
1526 assert((!RS || !LiveUnits) &&
"Only RS or LiveUnits can be set but not both");
1534 bool IsStore =
Desc->mayStore();
1535 bool IsFlat =
TII->isFLATScratch(LoadStoreOp);
1537 bool CanClobberSCC =
false;
1538 bool Scavenged =
false;
1548 unsigned EltSize = (IsFlat && !IsAGPR) ? std::min(RegWidth, 16u) : 4u;
1549 unsigned NumSubRegs = RegWidth / EltSize;
1550 unsigned Size = NumSubRegs * EltSize;
1551 unsigned RemSize = RegWidth -
Size;
1552 unsigned NumRemSubRegs = RemSize ? 1 : 0;
1554 int64_t MaterializedOffset =
Offset;
1556 int64_t MaxOffset =
Offset +
Size + RemSize - EltSize;
1557 int64_t ScratchOffsetRegDelta = 0;
1559 if (IsFlat && EltSize > 4) {
1561 Desc = &
TII->get(LoadStoreOp);
1568 "unexpected VGPR spill offset");
1575 bool UseVGPROffset =
false;
1582 if (IsFlat && SGPRBase) {
1606 bool IsOffsetLegal =
1609 :
TII->isLegalMUBUFImmOffset(MaxOffset);
1621 CanClobberSCC = !RS->
isRegUsed(AMDGPU::SCC);
1622 }
else if (LiveUnits) {
1623 CanClobberSCC = LiveUnits->
available(AMDGPU::SCC);
1624 for (
MCRegister Reg : AMDGPU::SGPR_32RegClass) {
1632 if (ScratchOffsetReg != AMDGPU::NoRegister && !CanClobberSCC)
1636 UseVGPROffset =
true;
1642 for (
MCRegister Reg : AMDGPU::VGPR_32RegClass) {
1644 TmpOffsetVGPR = Reg;
1651 }
else if (!SOffset && CanClobberSCC) {
1662 if (!ScratchOffsetReg)
1664 SOffset = ScratchOffsetReg;
1665 ScratchOffsetRegDelta =
Offset;
1673 if (!IsFlat && !UseVGPROffset)
1676 if (!UseVGPROffset && !SOffset)
1679 if (UseVGPROffset) {
1681 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR,
Offset);
1682 }
else if (ScratchOffsetReg == AMDGPU::NoRegister) {
1687 .
addReg(ScratchOffsetReg)
1689 Add->getOperand(3).setIsDead();
1695 if (IsFlat && SOffset == AMDGPU::NoRegister) {
1697 &&
"Unexpected vaddr for flat scratch with a FI operand");
1699 if (UseVGPROffset) {
1706 Desc = &
TII->get(LoadStoreOp);
1709 for (
unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e;
1710 ++i, RegOffset += EltSize) {
1711 if (i == NumSubRegs) {
1715 Desc = &
TII->get(LoadStoreOp);
1717 if (!IsFlat && UseVGPROffset) {
1720 Desc = &
TII->get(NewLoadStoreOp);
1723 if (UseVGPROffset && TmpOffsetVGPR == TmpIntermediateVGPR) {
1730 MaterializeVOffset(ScratchOffsetReg, TmpOffsetVGPR, MaterializedOffset);
1733 unsigned NumRegs = EltSize / 4;
1739 unsigned SOffsetRegState = 0;
1741 const bool IsLastSubReg = i + 1 == e;
1742 const bool IsFirstSubReg = i == 0;
1751 bool NeedSuperRegDef = e > 1 && IsStore && IsFirstSubReg;
1752 bool NeedSuperRegImpOperand = e > 1;
1756 unsigned RemEltSize = EltSize;
1764 for (
int LaneS = (RegOffset + EltSize) / 4 - 1, Lane = LaneS,
1765 LaneE = RegOffset / 4;
1766 Lane >= LaneE; --Lane) {
1767 bool IsSubReg = e > 1 || EltSize > 4;
1772 if (!MIB.getInstr())
1774 if (NeedSuperRegDef || (IsSubReg && IsStore && Lane == LaneS && IsFirstSubReg)) {
1776 NeedSuperRegDef =
false;
1778 if ((IsSubReg || NeedSuperRegImpOperand) && (IsFirstSubReg || IsLastSubReg)) {
1779 NeedSuperRegImpOperand =
true;
1780 unsigned State = SrcDstRegState;
1781 if (!IsLastSubReg || (Lane != LaneE))
1782 State &= ~RegState::Kill;
1783 if (!IsFirstSubReg || (Lane != LaneS))
1784 State &= ~RegState::Define;
1793 if (RemEltSize != EltSize) {
1794 assert(IsFlat && EltSize > 4);
1796 unsigned NumRegs = RemEltSize / 4;
1803 unsigned FinalReg =
SubReg;
1808 if (!TmpIntermediateVGPR) {
1814 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64),
1815 TmpIntermediateVGPR)
1817 if (NeedSuperRegDef)
1819 if (NeedSuperRegImpOperand && (IsFirstSubReg || IsLastSubReg))
1823 SubReg = TmpIntermediateVGPR;
1824 }
else if (UseVGPROffset) {
1825 if (!TmpOffsetVGPR) {
1841 if (UseVGPROffset) {
1850 if (SOffset == AMDGPU::NoRegister) {
1852 if (UseVGPROffset && ScratchOffsetReg) {
1853 MIB.
addReg(ScratchOffsetReg);
1860 MIB.addReg(SOffset, SOffsetRegState);
1863 MIB.addImm(
Offset + RegOffset);
1870 MIB.addMemOperand(NewMMO);
1872 if (!IsAGPR && NeedSuperRegDef)
1875 if (!IsStore && IsAGPR && TmpIntermediateVGPR != AMDGPU::NoRegister) {
1882 if (NeedSuperRegImpOperand && (IsFirstSubReg || IsLastSubReg))
1906 if (!IsStore &&
MI !=
MBB.
end() &&
MI->isReturn() &&
1909 MIB->tieOperands(0, MIB->getNumOperands() - 1);
1913 if (ScratchOffsetRegDelta != 0) {
1917 .
addImm(-ScratchOffsetRegDelta);
1923 bool IsKill)
const {
1941 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
1946 : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
1957 bool SpillToPhysVGPRLane)
const {
1958 assert(!
MI->getOperand(0).isUndef() &&
1959 "undef spill should have been deleted earlier");
1966 bool SpillToVGPR = !VGPRSpills.
empty();
1967 if (OnlyToVGPR && !SpillToVGPR)
1980 "Num of SGPRs spilled should be less than or equal to num of "
1983 for (
unsigned i = 0, e = SB.
NumSubRegs; i < e; ++i) {
1990 bool IsFirstSubreg = i == 0;
1992 bool UseKill = SB.
IsKill && IsLastSubreg;
1998 SB.
TII.get(AMDGPU::SI_SPILL_S32_TO_VGPR), Spill.VGPR)
2015 if (SB.
NumSubRegs > 1 && (IsFirstSubreg || IsLastSubreg))
2035 for (
unsigned i =
Offset * PVD.PerVGPR,
2045 SB.
TII.get(AMDGPU::SI_SPILL_S32_TO_VGPR), SB.
TmpVGPR)
2062 unsigned SuperKillState = 0;
2076 MI->eraseFromParent();
2088 bool SpillToPhysVGPRLane)
const {
2094 bool SpillToVGPR = !VGPRSpills.
empty();
2095 if (OnlyToVGPR && !SpillToVGPR)
2099 for (
unsigned i = 0, e = SB.
NumSubRegs; i < e; ++i) {
2107 SB.
TII.get(AMDGPU::SI_RESTORE_S32_FROM_VGPR),
SubReg)
2130 for (
unsigned i =
Offset * PVD.PerVGPR,
2138 bool LastSubReg = (i + 1 == e);
2140 SB.
TII.get(AMDGPU::SI_RESTORE_S32_FROM_VGPR),
SubReg)
2157 MI->eraseFromParent();
2177 for (
unsigned i =
Offset * PVD.PerVGPR,
2196 unsigned SuperKillState = 0;
2206 MI = RestoreMBB.
end();
2212 for (
unsigned i =
Offset * PVD.PerVGPR,
2219 bool LastSubReg = (i + 1 == e);
2240 switch (
MI->getOpcode()) {
2241 case AMDGPU::SI_SPILL_S1024_SAVE:
2242 case AMDGPU::SI_SPILL_S512_SAVE:
2243 case AMDGPU::SI_SPILL_S384_SAVE:
2244 case AMDGPU::SI_SPILL_S352_SAVE:
2245 case AMDGPU::SI_SPILL_S320_SAVE:
2246 case AMDGPU::SI_SPILL_S288_SAVE:
2247 case AMDGPU::SI_SPILL_S256_SAVE:
2248 case AMDGPU::SI_SPILL_S224_SAVE:
2249 case AMDGPU::SI_SPILL_S192_SAVE:
2250 case AMDGPU::SI_SPILL_S160_SAVE:
2251 case AMDGPU::SI_SPILL_S128_SAVE:
2252 case AMDGPU::SI_SPILL_S96_SAVE:
2253 case AMDGPU::SI_SPILL_S64_SAVE:
2254 case AMDGPU::SI_SPILL_S32_SAVE:
2255 return spillSGPR(
MI, FI, RS, Indexes, LIS,
true, SpillToPhysVGPRLane);
2256 case AMDGPU::SI_SPILL_S1024_RESTORE:
2257 case AMDGPU::SI_SPILL_S512_RESTORE:
2258 case AMDGPU::SI_SPILL_S384_RESTORE:
2259 case AMDGPU::SI_SPILL_S352_RESTORE:
2260 case AMDGPU::SI_SPILL_S320_RESTORE:
2261 case AMDGPU::SI_SPILL_S288_RESTORE:
2262 case AMDGPU::SI_SPILL_S256_RESTORE:
2263 case AMDGPU::SI_SPILL_S224_RESTORE:
2264 case AMDGPU::SI_SPILL_S192_RESTORE:
2265 case AMDGPU::SI_SPILL_S160_RESTORE:
2266 case AMDGPU::SI_SPILL_S128_RESTORE:
2267 case AMDGPU::SI_SPILL_S96_RESTORE:
2268 case AMDGPU::SI_SPILL_S64_RESTORE:
2269 case AMDGPU::SI_SPILL_S32_RESTORE:
2270 return restoreSGPR(
MI, FI, RS, Indexes, LIS,
true, SpillToPhysVGPRLane);
2277 int SPAdj,
unsigned FIOperandNum,
2286 assert(SPAdj == 0 &&
"unhandled SP adjustment in call sequence?");
2289 "unreserved scratch RSRC register");
2292 int Index =
MI->getOperand(FIOperandNum).getIndex();
2298 switch (
MI->getOpcode()) {
2300 case AMDGPU::SI_SPILL_S1024_SAVE:
2301 case AMDGPU::SI_SPILL_S512_SAVE:
2302 case AMDGPU::SI_SPILL_S384_SAVE:
2303 case AMDGPU::SI_SPILL_S352_SAVE:
2304 case AMDGPU::SI_SPILL_S320_SAVE:
2305 case AMDGPU::SI_SPILL_S288_SAVE:
2306 case AMDGPU::SI_SPILL_S256_SAVE:
2307 case AMDGPU::SI_SPILL_S224_SAVE:
2308 case AMDGPU::SI_SPILL_S192_SAVE:
2309 case AMDGPU::SI_SPILL_S160_SAVE:
2310 case AMDGPU::SI_SPILL_S128_SAVE:
2311 case AMDGPU::SI_SPILL_S96_SAVE:
2312 case AMDGPU::SI_SPILL_S64_SAVE:
2313 case AMDGPU::SI_SPILL_S32_SAVE: {
2318 case AMDGPU::SI_SPILL_S1024_RESTORE:
2319 case AMDGPU::SI_SPILL_S512_RESTORE:
2320 case AMDGPU::SI_SPILL_S384_RESTORE:
2321 case AMDGPU::SI_SPILL_S352_RESTORE:
2322 case AMDGPU::SI_SPILL_S320_RESTORE:
2323 case AMDGPU::SI_SPILL_S288_RESTORE:
2324 case AMDGPU::SI_SPILL_S256_RESTORE:
2325 case AMDGPU::SI_SPILL_S224_RESTORE:
2326 case AMDGPU::SI_SPILL_S192_RESTORE:
2327 case AMDGPU::SI_SPILL_S160_RESTORE:
2328 case AMDGPU::SI_SPILL_S128_RESTORE:
2329 case AMDGPU::SI_SPILL_S96_RESTORE:
2330 case AMDGPU::SI_SPILL_S64_RESTORE:
2331 case AMDGPU::SI_SPILL_S32_RESTORE: {
2336 case AMDGPU::SI_SPILL_V1024_SAVE:
2337 case AMDGPU::SI_SPILL_V512_SAVE:
2338 case AMDGPU::SI_SPILL_V384_SAVE:
2339 case AMDGPU::SI_SPILL_V352_SAVE:
2340 case AMDGPU::SI_SPILL_V320_SAVE:
2341 case AMDGPU::SI_SPILL_V288_SAVE:
2342 case AMDGPU::SI_SPILL_V256_SAVE:
2343 case AMDGPU::SI_SPILL_V224_SAVE:
2344 case AMDGPU::SI_SPILL_V192_SAVE:
2345 case AMDGPU::SI_SPILL_V160_SAVE:
2346 case AMDGPU::SI_SPILL_V128_SAVE:
2347 case AMDGPU::SI_SPILL_V96_SAVE:
2348 case AMDGPU::SI_SPILL_V64_SAVE:
2349 case AMDGPU::SI_SPILL_V32_SAVE:
2350 case AMDGPU::SI_SPILL_A1024_SAVE:
2351 case AMDGPU::SI_SPILL_A512_SAVE:
2352 case AMDGPU::SI_SPILL_A384_SAVE:
2353 case AMDGPU::SI_SPILL_A352_SAVE:
2354 case AMDGPU::SI_SPILL_A320_SAVE:
2355 case AMDGPU::SI_SPILL_A288_SAVE:
2356 case AMDGPU::SI_SPILL_A256_SAVE:
2357 case AMDGPU::SI_SPILL_A224_SAVE:
2358 case AMDGPU::SI_SPILL_A192_SAVE:
2359 case AMDGPU::SI_SPILL_A160_SAVE:
2360 case AMDGPU::SI_SPILL_A128_SAVE:
2361 case AMDGPU::SI_SPILL_A96_SAVE:
2362 case AMDGPU::SI_SPILL_A64_SAVE:
2363 case AMDGPU::SI_SPILL_A32_SAVE:
2364 case AMDGPU::SI_SPILL_AV1024_SAVE:
2365 case AMDGPU::SI_SPILL_AV512_SAVE:
2366 case AMDGPU::SI_SPILL_AV384_SAVE:
2367 case AMDGPU::SI_SPILL_AV352_SAVE:
2368 case AMDGPU::SI_SPILL_AV320_SAVE:
2369 case AMDGPU::SI_SPILL_AV288_SAVE:
2370 case AMDGPU::SI_SPILL_AV256_SAVE:
2371 case AMDGPU::SI_SPILL_AV224_SAVE:
2372 case AMDGPU::SI_SPILL_AV192_SAVE:
2373 case AMDGPU::SI_SPILL_AV160_SAVE:
2374 case AMDGPU::SI_SPILL_AV128_SAVE:
2375 case AMDGPU::SI_SPILL_AV96_SAVE:
2376 case AMDGPU::SI_SPILL_AV64_SAVE:
2377 case AMDGPU::SI_SPILL_AV32_SAVE:
2378 case AMDGPU::SI_SPILL_WWM_V32_SAVE:
2379 case AMDGPU::SI_SPILL_WWM_AV32_SAVE: {
2381 AMDGPU::OpName::vdata);
2383 MI->eraseFromParent();
2387 assert(
TII->getNamedOperand(*
MI, AMDGPU::OpName::soffset)->getReg() ==
2391 : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
2392 auto *
MBB =
MI->getParent();
2393 bool IsWWMRegSpill =
TII->isWWMRegSpillOpcode(
MI->getOpcode());
2394 if (IsWWMRegSpill) {
2400 TII->getNamedOperand(*
MI, AMDGPU::OpName::offset)->getImm(),
2401 *
MI->memoperands_begin(), RS);
2406 MI->eraseFromParent();
2409 case AMDGPU::SI_SPILL_V32_RESTORE:
2410 case AMDGPU::SI_SPILL_V64_RESTORE:
2411 case AMDGPU::SI_SPILL_V96_RESTORE:
2412 case AMDGPU::SI_SPILL_V128_RESTORE:
2413 case AMDGPU::SI_SPILL_V160_RESTORE:
2414 case AMDGPU::SI_SPILL_V192_RESTORE:
2415 case AMDGPU::SI_SPILL_V224_RESTORE:
2416 case AMDGPU::SI_SPILL_V256_RESTORE:
2417 case AMDGPU::SI_SPILL_V288_RESTORE:
2418 case AMDGPU::SI_SPILL_V320_RESTORE:
2419 case AMDGPU::SI_SPILL_V352_RESTORE:
2420 case AMDGPU::SI_SPILL_V384_RESTORE:
2421 case AMDGPU::SI_SPILL_V512_RESTORE:
2422 case AMDGPU::SI_SPILL_V1024_RESTORE:
2423 case AMDGPU::SI_SPILL_A32_RESTORE:
2424 case AMDGPU::SI_SPILL_A64_RESTORE:
2425 case AMDGPU::SI_SPILL_A96_RESTORE:
2426 case AMDGPU::SI_SPILL_A128_RESTORE:
2427 case AMDGPU::SI_SPILL_A160_RESTORE:
2428 case AMDGPU::SI_SPILL_A192_RESTORE:
2429 case AMDGPU::SI_SPILL_A224_RESTORE:
2430 case AMDGPU::SI_SPILL_A256_RESTORE:
2431 case AMDGPU::SI_SPILL_A288_RESTORE:
2432 case AMDGPU::SI_SPILL_A320_RESTORE:
2433 case AMDGPU::SI_SPILL_A352_RESTORE:
2434 case AMDGPU::SI_SPILL_A384_RESTORE:
2435 case AMDGPU::SI_SPILL_A512_RESTORE:
2436 case AMDGPU::SI_SPILL_A1024_RESTORE:
2437 case AMDGPU::SI_SPILL_AV32_RESTORE:
2438 case AMDGPU::SI_SPILL_AV64_RESTORE:
2439 case AMDGPU::SI_SPILL_AV96_RESTORE:
2440 case AMDGPU::SI_SPILL_AV128_RESTORE:
2441 case AMDGPU::SI_SPILL_AV160_RESTORE:
2442 case AMDGPU::SI_SPILL_AV192_RESTORE:
2443 case AMDGPU::SI_SPILL_AV224_RESTORE:
2444 case AMDGPU::SI_SPILL_AV256_RESTORE:
2445 case AMDGPU::SI_SPILL_AV288_RESTORE:
2446 case AMDGPU::SI_SPILL_AV320_RESTORE:
2447 case AMDGPU::SI_SPILL_AV352_RESTORE:
2448 case AMDGPU::SI_SPILL_AV384_RESTORE:
2449 case AMDGPU::SI_SPILL_AV512_RESTORE:
2450 case AMDGPU::SI_SPILL_AV1024_RESTORE:
2451 case AMDGPU::SI_SPILL_WWM_V32_RESTORE:
2452 case AMDGPU::SI_SPILL_WWM_AV32_RESTORE: {
2454 AMDGPU::OpName::vdata);
2455 assert(
TII->getNamedOperand(*
MI, AMDGPU::OpName::soffset)->getReg() ==
2459 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
2460 auto *
MBB =
MI->getParent();
2461 bool IsWWMRegSpill =
TII->isWWMRegSpillOpcode(
MI->getOpcode());
2462 if (IsWWMRegSpill) {
2469 TII->getNamedOperand(*
MI, AMDGPU::OpName::offset)->getImm(),
2470 *
MI->memoperands_begin(), RS);
2475 MI->eraseFromParent();
2478 case AMDGPU::V_ADD_U32_e32:
2479 case AMDGPU::V_ADD_U32_e64:
2480 case AMDGPU::V_ADD_CO_U32_e32:
2481 case AMDGPU::V_ADD_CO_U32_e64: {
2483 unsigned NumDefs =
MI->getNumExplicitDefs();
2484 unsigned Src0Idx = NumDefs;
2486 bool HasClamp =
false;
2489 switch (
MI->getOpcode()) {
2490 case AMDGPU::V_ADD_U32_e32:
2492 case AMDGPU::V_ADD_U32_e64:
2493 HasClamp =
MI->getOperand(3).getImm();
2495 case AMDGPU::V_ADD_CO_U32_e32:
2496 VCCOp = &
MI->getOperand(3);
2498 case AMDGPU::V_ADD_CO_U32_e64:
2499 VCCOp = &
MI->getOperand(1);
2500 HasClamp =
MI->getOperand(4).getImm();
2505 bool DeadVCC = !VCCOp || VCCOp->
isDead();
2509 unsigned OtherOpIdx =
2510 FIOperandNum == Src0Idx ? FIOperandNum + 1 : Src0Idx;
2513 unsigned Src1Idx = Src0Idx + 1;
2514 Register MaterializedReg = FrameReg;
2517 int64_t
Offset = FrameInfo.getObjectOffset(Index);
2521 if (OtherOp->
isImm()) {
2532 OtherOp->
setImm(TotalOffset);
2545 AMDGPU::VGPR_32RegClass,
MI,
false, 0);
2553 MaterializedReg = ScavengedVGPR;
2556 if ((!OtherOp->
isImm() || OtherOp->
getImm() != 0) && MaterializedReg) {
2558 !
TII->isOperandLegal(*
MI, Src1Idx, OtherOp)) {
2565 if (!ScavengedVGPR) {
2567 AMDGPU::VGPR_32RegClass,
MI,
false,
2571 assert(ScavengedVGPR != DstReg);
2576 MaterializedReg = ScavengedVGPR;
2585 AddI32.
add(
MI->getOperand(1));
2587 unsigned MaterializedRegFlags =
2590 if (
isVGPRClass(getPhysRegBaseClass(MaterializedReg))) {
2595 .addReg(MaterializedReg, MaterializedRegFlags);
2600 .addReg(MaterializedReg, MaterializedRegFlags)
2604 if (
MI->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 ||
2605 MI->getOpcode() == AMDGPU::V_ADD_U32_e64)
2608 if (
MI->getOpcode() == AMDGPU::V_ADD_CO_U32_e32)
2609 AddI32.setOperandDead(3);
2611 MaterializedReg = DstReg;
2617 }
else if (
Offset != 0) {
2618 assert(!MaterializedReg);
2622 if (DeadVCC && !HasClamp) {
2627 if (OtherOp->
isReg() && OtherOp->
getReg() == DstReg) {
2629 MI->eraseFromParent();
2634 MI->setDesc(
TII->get(AMDGPU::V_MOV_B32_e32));
2635 MI->removeOperand(FIOperandNum);
2637 unsigned NumOps =
MI->getNumOperands();
2638 for (
unsigned I = NumOps - 2;
I >= NumDefs + 1; --
I)
2639 MI->removeOperand(
I);
2642 MI->removeOperand(1);
2654 if (!
TII->isOperandLegal(*
MI, Src1Idx) &&
TII->commuteInstruction(*
MI)) {
2662 for (
unsigned SrcIdx : {FIOperandNum, OtherOpIdx}) {
2663 if (!
TII->isOperandLegal(*
MI, SrcIdx)) {
2667 if (!ScavengedVGPR) {
2669 AMDGPU::VGPR_32RegClass,
MI,
false,
2673 assert(ScavengedVGPR != DstReg);
2679 Src.ChangeToRegister(ScavengedVGPR,
false);
2680 Src.setIsKill(
true);
2686 if (FIOp->
isImm() && FIOp->
getImm() == 0 && DeadVCC && !HasClamp) {
2687 if (OtherOp->
isReg() && OtherOp->
getReg() != DstReg) {
2691 MI->eraseFromParent();
2696 case AMDGPU::S_ADD_I32: {
2698 unsigned OtherOpIdx = FIOperandNum == 1 ? 2 : 1;
2705 Register MaterializedReg = FrameReg;
2708 bool DeadSCC =
MI->getOperand(3).isDead();
2729 MaterializedReg = TmpReg;
2732 int64_t
Offset = FrameInfo.getObjectOffset(Index);
2737 if (OtherOp.
isImm()) {
2741 if (MaterializedReg)
2745 }
else if (MaterializedReg) {
2749 if (!TmpReg && MaterializedReg == FrameReg) {
2763 MaterializedReg = DstReg;
2775 if (DeadSCC && OtherOp.
isImm() && OtherOp.
getImm() == 0) {
2777 MI->removeOperand(3);
2778 MI->removeOperand(OtherOpIdx);
2779 MI->setDesc(
TII->get(FIOp->
isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
2780 }
else if (DeadSCC && FIOp->
isImm() && FIOp->
getImm() == 0) {
2782 MI->removeOperand(3);
2783 MI->removeOperand(FIOperandNum);
2785 TII->get(OtherOp.
isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32));
2796 int64_t
Offset = FrameInfo.getObjectOffset(Index);
2798 if (
TII->isFLATScratch(*
MI)) {
2800 (int16_t)FIOperandNum ==
2808 TII->getNamedOperand(*
MI, AMDGPU::OpName::offset);
2812 OffsetOp->
setImm(NewOffset);
2819 unsigned Opc =
MI->getOpcode();
2834 bool TiedVDst = VDstIn != -1 &&
MI->getOperand(VDstIn).isReg() &&
2835 MI->getOperand(VDstIn).isTied();
2837 MI->untieRegOperand(VDstIn);
2847 assert(NewVDst != -1 && NewVDstIn != -1 &&
"Must be tied!");
2848 MI->tieOperands(NewVDst, NewVDstIn);
2850 MI->setDesc(
TII->get(NewOpc));
2858 if (
TII->isImmOperandLegal(*
MI, FIOperandNum, *FIOp))
2865 bool UseSGPR =
TII->isOperandLegal(*
MI, FIOperandNum, FIOp);
2867 if (!
Offset && FrameReg && UseSGPR) {
2873 UseSGPR ? &AMDGPU::SReg_32_XM0RegClass : &AMDGPU::VGPR_32RegClass;
2880 if ((!FrameReg || !
Offset) && TmpReg) {
2881 unsigned Opc = UseSGPR ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
2884 MIB.addReg(FrameReg);
2891 bool NeedSaveSCC = RS->
isRegUsed(AMDGPU::SCC) &&
2892 !
MI->definesRegister(AMDGPU::SCC,
nullptr);
2897 MI,
false, 0, !UseSGPR);
2901 if ((!TmpSReg && !FrameReg) || (!TmpReg && !UseSGPR))
2912 assert(!(
Offset & 0x1) &&
"Flat scratch offset must be aligned!");
2932 if (TmpSReg == FrameReg) {
2935 !
MI->registerDefIsDead(AMDGPU::SCC,
nullptr)) {
2959 bool IsMUBUF =
TII->isMUBUF(*
MI);
2965 bool LiveSCC = RS->
isRegUsed(AMDGPU::SCC) &&
2966 !
MI->definesRegister(AMDGPU::SCC,
nullptr);
2968 ? &AMDGPU::SReg_32RegClass
2969 : &AMDGPU::VGPR_32RegClass;
2970 bool IsCopy =
MI->getOpcode() == AMDGPU::V_MOV_B32_e32 ||
2971 MI->getOpcode() == AMDGPU::V_MOV_B32_e64 ||
2972 MI->getOpcode() == AMDGPU::S_MOV_B32;
2974 IsCopy ?
MI->getOperand(0).getReg()
2977 int64_t
Offset = FrameInfo.getObjectOffset(Index);
2980 IsSALU && !LiveSCC ? AMDGPU::S_LSHR_B32 : AMDGPU::V_LSHRREV_B32_e64;
2982 if (IsSALU && LiveSCC) {
2988 if (OpCode == AMDGPU::V_LSHRREV_B32_e64)
2994 if (IsSALU && !LiveSCC)
2995 Shift.getInstr()->getOperand(3).setIsDead();
2996 if (IsSALU && LiveSCC) {
3003 ResultReg = NewDest;
3008 if ((MIB =
TII->getAddNoCarry(*
MBB,
MI,
DL, ResultReg, *RS)) !=
3018 const bool IsVOP2 = MIB->
getOpcode() == AMDGPU::V_ADD_U32_e32;
3030 "Need to reuse carry out register");
3035 ConstOffsetReg = getSubReg(MIB.
getReg(1), AMDGPU::sub0);
3037 ConstOffsetReg = MIB.
getReg(1);
3048 if (!MIB || IsSALU) {
3055 Register TmpScaledReg = IsCopy && IsSALU
3058 AMDGPU::SReg_32_XM0RegClass,
MI,
3060 Register ScaledReg = TmpScaledReg.
isValid() ? TmpScaledReg : FrameReg;
3072 AMDGPU::VGPR_32RegClass,
MI,
false, 0,
true);
3075 if ((
Add =
TII->getAddNoCarry(*
MBB,
MI,
DL, TmpResultReg, *RS))) {
3080 if (
Add->getOpcode() == AMDGPU::V_ADD_CO_U32_e64) {
3090 "offset is unsafe for v_mad_u32_u24");
3099 bool IsInlinableLiteral =
3101 if (!IsInlinableLiteral) {
3110 if (!IsInlinableLiteral) {
3123 Register NewDest = IsCopy ? ResultReg
3125 AMDGPU::SReg_32RegClass, *
Add,
3130 ResultReg = NewDest;
3136 ResultReg = TmpResultReg;
3138 if (!TmpScaledReg.
isValid()) {
3151 MI->eraseFromParent();
3161 static_cast<int>(FIOperandNum) ==
3164 auto &SOffset = *
TII->getNamedOperand(*
MI, AMDGPU::OpName::soffset);
3165 assert((SOffset.isImm() && SOffset.getImm() == 0));
3167 if (FrameReg != AMDGPU::NoRegister)
3168 SOffset.ChangeToRegister(FrameReg,
false);
3170 int64_t
Offset = FrameInfo.getObjectOffset(Index);
3172 TII->getNamedOperand(*
MI, AMDGPU::OpName::offset)->getImm();
3173 int64_t NewOffset = OldImm +
Offset;
3175 if (
TII->isLegalMUBUFImmOffset(NewOffset) &&
3177 MI->eraseFromParent();
3186 if (!
TII->isImmOperandLegal(*
MI, FIOperandNum, *FIOp)) {
3208 return &AMDGPU::VReg_64RegClass;
3210 return &AMDGPU::VReg_96RegClass;
3212 return &AMDGPU::VReg_128RegClass;
3214 return &AMDGPU::VReg_160RegClass;
3216 return &AMDGPU::VReg_192RegClass;
3218 return &AMDGPU::VReg_224RegClass;
3220 return &AMDGPU::VReg_256RegClass;
3222 return &AMDGPU::VReg_288RegClass;
3224 return &AMDGPU::VReg_320RegClass;
3226 return &AMDGPU::VReg_352RegClass;
3228 return &AMDGPU::VReg_384RegClass;
3230 return &AMDGPU::VReg_512RegClass;
3232 return &AMDGPU::VReg_1024RegClass;
3240 return &AMDGPU::VReg_64_Align2RegClass;
3242 return &AMDGPU::VReg_96_Align2RegClass;
3244 return &AMDGPU::VReg_128_Align2RegClass;
3246 return &AMDGPU::VReg_160_Align2RegClass;
3248 return &AMDGPU::VReg_192_Align2RegClass;
3250 return &AMDGPU::VReg_224_Align2RegClass;
3252 return &AMDGPU::VReg_256_Align2RegClass;
3254 return &AMDGPU::VReg_288_Align2RegClass;
3256 return &AMDGPU::VReg_320_Align2RegClass;
3258 return &AMDGPU::VReg_352_Align2RegClass;
3260 return &AMDGPU::VReg_384_Align2RegClass;
3262 return &AMDGPU::VReg_512_Align2RegClass;
3264 return &AMDGPU::VReg_1024_Align2RegClass;
3272 return &AMDGPU::VReg_1RegClass;
3274 return &AMDGPU::VGPR_16RegClass;
3276 return &AMDGPU::VGPR_32RegClass;
3284 return &AMDGPU::AReg_64RegClass;
3286 return &AMDGPU::AReg_96RegClass;
3288 return &AMDGPU::AReg_128RegClass;
3290 return &AMDGPU::AReg_160RegClass;
3292 return &AMDGPU::AReg_192RegClass;
3294 return &AMDGPU::AReg_224RegClass;
3296 return &AMDGPU::AReg_256RegClass;
3298 return &AMDGPU::AReg_288RegClass;
3300 return &AMDGPU::AReg_320RegClass;
3302 return &AMDGPU::AReg_352RegClass;
3304 return &AMDGPU::AReg_384RegClass;
3306 return &AMDGPU::AReg_512RegClass;
3308 return &AMDGPU::AReg_1024RegClass;
3316 return &AMDGPU::AReg_64_Align2RegClass;
3318 return &AMDGPU::AReg_96_Align2RegClass;
3320 return &AMDGPU::AReg_128_Align2RegClass;
3322 return &AMDGPU::AReg_160_Align2RegClass;
3324 return &AMDGPU::AReg_192_Align2RegClass;
3326 return &AMDGPU::AReg_224_Align2RegClass;
3328 return &AMDGPU::AReg_256_Align2RegClass;
3330 return &AMDGPU::AReg_288_Align2RegClass;
3332 return &AMDGPU::AReg_320_Align2RegClass;
3334 return &AMDGPU::AReg_352_Align2RegClass;
3336 return &AMDGPU::AReg_384_Align2RegClass;
3338 return &AMDGPU::AReg_512_Align2RegClass;
3340 return &AMDGPU::AReg_1024_Align2RegClass;
3348 return &AMDGPU::AGPR_LO16RegClass;
3350 return &AMDGPU::AGPR_32RegClass;
3358 return &AMDGPU::AV_64RegClass;
3360 return &AMDGPU::AV_96RegClass;
3362 return &AMDGPU::AV_128RegClass;
3364 return &AMDGPU::AV_160RegClass;
3366 return &AMDGPU::AV_192RegClass;
3368 return &AMDGPU::AV_224RegClass;
3370 return &AMDGPU::AV_256RegClass;
3372 return &AMDGPU::AV_288RegClass;
3374 return &AMDGPU::AV_320RegClass;
3376 return &AMDGPU::AV_352RegClass;
3378 return &AMDGPU::AV_384RegClass;
3380 return &AMDGPU::AV_512RegClass;
3382 return &AMDGPU::AV_1024RegClass;
3390 return &AMDGPU::AV_64_Align2RegClass;
3392 return &AMDGPU::AV_96_Align2RegClass;
3394 return &AMDGPU::AV_128_Align2RegClass;
3396 return &AMDGPU::AV_160_Align2RegClass;
3398 return &AMDGPU::AV_192_Align2RegClass;
3400 return &AMDGPU::AV_224_Align2RegClass;
3402 return &AMDGPU::AV_256_Align2RegClass;
3404 return &AMDGPU::AV_288_Align2RegClass;
3406 return &AMDGPU::AV_320_Align2RegClass;
3408 return &AMDGPU::AV_352_Align2RegClass;
3410 return &AMDGPU::AV_384_Align2RegClass;
3412 return &AMDGPU::AV_512_Align2RegClass;
3414 return &AMDGPU::AV_1024_Align2RegClass;
3422 return &AMDGPU::AV_32RegClass;
3431 return &AMDGPU::SGPR_LO16RegClass;
3433 return &AMDGPU::SReg_32RegClass;
3435 return &AMDGPU::SReg_64RegClass;
3437 return &AMDGPU::SGPR_96RegClass;
3439 return &AMDGPU::SGPR_128RegClass;
3441 return &AMDGPU::SGPR_160RegClass;
3443 return &AMDGPU::SGPR_192RegClass;
3445 return &AMDGPU::SGPR_224RegClass;
3447 return &AMDGPU::SGPR_256RegClass;
3449 return &AMDGPU::SGPR_288RegClass;
3451 return &AMDGPU::SGPR_320RegClass;
3453 return &AMDGPU::SGPR_352RegClass;
3455 return &AMDGPU::SGPR_384RegClass;
3457 return &AMDGPU::SGPR_512RegClass;
3459 return &AMDGPU::SGPR_1024RegClass;
3467 if (Reg.isVirtual())
3468 RC =
MRI.getRegClass(Reg);
3470 RC = getPhysRegBaseClass(Reg);
3476 unsigned Size = getRegSizeInBits(*SRC);
3478 assert(VRC &&
"Invalid register class size");
3484 unsigned Size = getRegSizeInBits(*SRC);
3486 assert(ARC &&
"Invalid register class size");
3492 unsigned Size = getRegSizeInBits(*VRC);
3494 return &AMDGPU::SGPR_32RegClass;
3496 assert(SRC &&
"Invalid register class size");
3503 unsigned SubIdx)
const {
3506 getMatchingSuperRegClass(SuperRC, SubRC, SubIdx);
3507 return MatchRC && MatchRC->
hasSubClassEq(SuperRC) ? MatchRC :
nullptr;
3523 unsigned SrcSubReg)
const {
3540 return getCommonSubClass(DefRC, SrcRC) !=
nullptr;
3556 if (ReserveHighestRegister) {
3558 if (
MRI.isAllocatable(Reg) && !
MRI.isPhysRegUsed(Reg))
3562 if (
MRI.isAllocatable(Reg) && !
MRI.isPhysRegUsed(Reg))
3579 unsigned EltSize)
const {
3581 assert(RegBitWidth >= 32 && RegBitWidth <= 1024);
3583 const unsigned RegDWORDs = RegBitWidth / 32;
3584 const unsigned EltDWORDs = EltSize / 4;
3585 assert(RegSplitParts.size() + 1 >= EltDWORDs);
3587 const std::vector<int16_t> &Parts = RegSplitParts[EltDWORDs - 1];
3588 const unsigned NumParts = RegDWORDs / EltDWORDs;
3590 return ArrayRef(Parts.data(), NumParts);
3596 return Reg.isVirtual() ?
MRI.getRegClass(Reg) : getPhysRegBaseClass(Reg);
3603 return getSubRegisterClass(SrcRC, MO.
getSubReg());
3628 unsigned SrcSize = getRegSizeInBits(*SrcRC);
3629 unsigned DstSize = getRegSizeInBits(*DstRC);
3630 unsigned NewSize = getRegSizeInBits(*NewRC);
3636 if (SrcSize <= 32 || DstSize <= 32)
3639 return NewSize <= DstSize || NewSize <= SrcSize;
3645 switch (RC->
getID()) {
3647 return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF);
3648 case AMDGPU::VGPR_32RegClassID:
3650 case AMDGPU::SGPR_32RegClassID:
3651 case AMDGPU::SGPR_LO16RegClassID:
3657 unsigned Idx)
const {
3658 if (
Idx == AMDGPU::RegisterPressureSets::VGPR_32 ||
3659 Idx == AMDGPU::RegisterPressureSets::AGPR_32)
3663 if (
Idx == AMDGPU::RegisterPressureSets::SReg_32)
3671 static const int Empty[] = { -1 };
3673 if (RegPressureIgnoredUnits[RegUnit])
3676 return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit);
3681 return AMDGPU::SGPR30_SGPR31;
3687 switch (RB.
getID()) {
3688 case AMDGPU::VGPRRegBankID:
3691 case AMDGPU::VCCRegBankID:
3694 case AMDGPU::SGPRRegBankID:
3696 case AMDGPU::AGPRRegBankID:
3707 if (
const RegisterBank *RB = dyn_cast<const RegisterBank *>(RCOrRB))
3710 if (
const auto *RC = dyn_cast<const TargetRegisterClass *>(RCOrRB))
3711 return getAllocatableClass(RC);
3717 return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC;
3721 return isWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
3727 : &AMDGPU::VReg_64RegClass;
3732 switch ((
int)RCID) {
3733 case AMDGPU::SReg_1RegClassID:
3735 case AMDGPU::SReg_1_XEXECRegClassID:
3740 return AMDGPUGenRegisterInfo::getRegClass(RCID);
3753 if (Reg.isVirtual()) {
3758 :
MRI.getMaxLaneMaskForVReg(Reg);
3762 if ((S.LaneMask & SubLanes) == SubLanes) {
3763 V = S.getVNInfoAt(UseIdx);
3775 for (
MCRegUnit Unit : regunits(Reg.asMCReg())) {
3790 if (!Def || !MDT.dominates(Def, &
Use))
3793 assert(Def->modifiesRegister(Reg,
this));
3799 assert(getRegSizeInBits(*getPhysRegBaseClass(Reg)) <= 32);
3802 AMDGPU::SReg_32RegClass,
3803 AMDGPU::AGPR_32RegClass } ) {
3804 if (
MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::lo16, &RC))
3807 if (
MCPhysReg Super = getMatchingSuperReg(Reg, AMDGPU::hi16,
3808 &AMDGPU::VGPR_32RegClass)) {
3812 return AMDGPU::NoRegister;
3835 unsigned Size = getRegSizeInBits(*RC);
3869 return std::min(128u, getSubRegIdxSize(
SubReg));
3873 return std::min(32u, getSubRegIdxSize(
SubReg));
3884 if (
MRI.isPhysRegUsed(Reg))
unsigned const MachineRegisterInfo * MRI
Provides AMDGPU specific target descriptions.
This file declares the targeting of the RegisterBankInfo class for AMDGPU.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
Analysis containing CSE Info
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static int getOffenMUBUFStore(unsigned Opc)
static const TargetRegisterClass * getAnyAGPRClassForBitWidth(unsigned BitWidth)
static int getOffsetMUBUFLoad(unsigned Opc)
static const std::array< unsigned, 17 > SubRegFromChannelTableWidthMap
static void emitUnsupportedError(const Function &Fn, const MachineInstr &MI, const Twine &ErrMsg)
static const TargetRegisterClass * getAlignedAGPRClassForBitWidth(unsigned BitWidth)
static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST, MachineFrameInfo &MFI, MachineBasicBlock::iterator MI, int Index, int64_t Offset)
static unsigned getFlatScratchSpillOpcode(const SIInstrInfo *TII, unsigned LoadStoreOp, unsigned EltSize)
static const TargetRegisterClass * getAlignedVGPRClassForBitWidth(unsigned BitWidth)
static int getOffsetMUBUFStore(unsigned Opc)
static const TargetRegisterClass * getAnyVGPRClassForBitWidth(unsigned BitWidth)
static cl::opt< bool > EnableSpillSGPRToVGPR("amdgpu-spill-sgpr-to-vgpr", cl::desc("Enable spilling SGPRs to VGPRs"), cl::ReallyHidden, cl::init(true))
static unsigned getNumSubRegsForSpillOp(unsigned Op)
static const TargetRegisterClass * getAlignedVectorSuperClassForBitWidth(unsigned BitWidth)
static const TargetRegisterClass * getAnyVectorSuperClassForBitWidth(unsigned BitWidth)
static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, int Index, unsigned Lane, unsigned ValueReg, bool IsKill)
static bool isFIPlusImmOrVGPR(const SIRegisterInfo &TRI, const MachineInstr &MI)
static int getOffenMUBUFLoad(unsigned Opc)
Interface definition for SIRegisterInfo.
static const char * getRegisterName(MCRegister Reg)
bool isBottomOfStack() const
std::pair< unsigned, unsigned > getOccupancyWithWorkGroupSizes(uint32_t LDSBytes, const Function &F) const
Subtarget's minimum/maximum occupancy, in number of waves per EU, that can be achieved when the only ...
bool useRealTrue16Insts() const
Return true if real (non-fake) variants of True16 instructions using 16-bit registers should be code-...
unsigned getWavefrontSizeLog2() const
unsigned getWavefrontSize() const
bool hasInv2PiInlineImm() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
bool test(unsigned Idx) const
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
bool empty() const
empty - Tests whether there are no bits in this bitvector.
This class represents an Operation in the Expression.
Diagnostic information for unsupported feature in backend.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasGFX90AInsts() const
bool hasMFMAInlineLiteralBug() const
const SIInstrInfo * getInstrInfo() const override
unsigned getConstantBusLimit(unsigned Opcode) const
bool needsAlignedVGPRs() const
Return if operations acting on VGPR tuples require even alignment.
bool enableFlatScratch() const
unsigned getMaxNumVGPRs(unsigned WavesPerEU) const
const SIFrameLowering * getFrameLowering() const override
unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const
bool hasVOP3Literal() const
bool hasFlatScratchSTMode() const
unsigned getMaxWaveScratchSize() const
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
LiveInterval - This class represents the liveness of a register, or stack slot.
bool hasSubRanges() const
Returns true if subregister liveness information is available.
iterator_range< subrange_iterator > subranges()
void removeAllRegUnitsForPhysReg(MCRegister Reg)
Remove associated live ranges for the register units associated with Reg.
bool hasInterval(Register Reg) const
MachineInstr * getInstructionFromIndex(SlotIndex index) const
Returns the instruction associated with the given index.
MachineDominatorTree & getDomTree()
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
LiveRange & getRegUnit(unsigned Unit)
Return the live range for register unit Unit.
LiveInterval & getInterval(Register Reg)
This class represents the liveness of a register, stack slot, etc.
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
A set of register units used to track register liveness.
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
Describe properties that are true of each instruction in the target description file.
MCRegAliasIterator enumerates all registers aliasing Reg.
Wrapper class representing physical registers. Should be passed by value.
static MCRegister from(unsigned Val)
Check the provided unsigned value is a valid MCRegister.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasCalls() const
Return true if the current function has any function calls.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool hasStackObjects() const
Return true if there are any stack objects in this function.
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
void setAsmPrinterFlag(uint8_t Flag)
Set a flag for the AsmPrinter.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
void setImm(int64_t immVal)
void setIsRenamable(bool Val=true)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setIsDead(bool Val=true)
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
void setIsKill(bool Val=true)
void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value.
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool isReserved(MCRegister PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
void assignRegToScavengingIndex(int FI, Register Reg, MachineInstr *Restore=nullptr)
Record that Reg is in use at scavenging index FI.
Register scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, bool RestoreAfter, int SPAdj, bool AllowSpill=true)
Make a register of the specific register class available from the current position backwards to the p...
Holds all the information related to register banks.
virtual bool isDivergentRegBank(const RegisterBank *RB) const
Returns true if the register bank is considered divergent.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
static bool isVOP3(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isMUBUF(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool usesAGPRs(const MachineFunction &MF) const
ArrayRef< MCPhysReg > getAGPRSpillVGPRs() const
MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const
Register getLongBranchReservedReg() const
Register getStackPtrOffsetReg() const
Register getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses.
ArrayRef< MCPhysReg > getVGPRSpillAGPRs() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
ArrayRef< SIRegisterInfo::SpilledReg > getSGPRSpillToVirtualVGPRLanes(int FrameIndex) const
Register getSGPRForEXECCopy() const
ArrayRef< SIRegisterInfo::SpilledReg > getSGPRSpillToPhysicalVGPRLanes(int FrameIndex) const
Register getVGPRForAGPRCopy() const
Register getFrameOffsetReg() const
BitVector getNonWWMRegMask() const
bool checkFlag(Register Reg, uint8_t Flag) const
void addToSpilledVGPRs(unsigned num)
const ReservedRegSet & getWWMReservedRegs() const
void addToSpilledSGPRs(unsigned num)
Register materializeFrameBaseRegister(MachineBasicBlock *MBB, int FrameIdx, int64_t Offset) const override
int64_t getScratchInstrOffset(const MachineInstr *MI) const
bool isFrameOffsetLegal(const MachineInstr *MI, Register BaseReg, int64_t Offset) const override
const TargetRegisterClass * getRegClass(unsigned RCID) const
const TargetRegisterClass * getCompatibleSubRegClass(const TargetRegisterClass *SuperRC, const TargetRegisterClass *SubRC, unsigned SubIdx) const
Returns a register class which is compatible with SuperRC, such that a subregister exists with class ...
ArrayRef< MCPhysReg > getAllSGPR64(const MachineFunction &MF) const
Return all SGPR64 which satisfy the waves per execution unit requirement of the subtarget.
MCRegister findUnusedRegister(const MachineRegisterInfo &MRI, const TargetRegisterClass *RC, const MachineFunction &MF, bool ReserveHighestVGPR=false) const
Returns a lowest register that is not used at any point in the function.
static unsigned getSubRegFromChannel(unsigned Channel, unsigned NumRegs=1)
MCPhysReg get32BitRegister(MCPhysReg Reg) const
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
bool requiresFrameIndexReplacementScavenging(const MachineFunction &MF) const override
const TargetRegisterClass * getProperlyAlignedRC(const TargetRegisterClass *RC) const
bool shouldRealignStack(const MachineFunction &MF) const override
bool restoreSGPR(MachineBasicBlock::iterator MI, int FI, RegScavenger *RS, SlotIndexes *Indexes=nullptr, LiveIntervals *LIS=nullptr, bool OnlyToVGPR=false, bool SpillToPhysVGPRLane=false) const
bool isProperlyAlignedRC(const TargetRegisterClass &RC) const
const TargetRegisterClass * getEquivalentVGPRClass(const TargetRegisterClass *SRC) const
Register getFrameRegister(const MachineFunction &MF) const override
LLVM_READONLY const TargetRegisterClass * getVectorSuperClassForBitWidth(unsigned BitWidth) const
bool spillEmergencySGPR(MachineBasicBlock::iterator MI, MachineBasicBlock &RestoreMBB, Register SGPR, RegScavenger *RS) const
SIRegisterInfo(const GCNSubtarget &ST)
const uint32_t * getAllVGPRRegMask() const
MCRegister getReturnAddressReg(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
bool hasBasePointer(const MachineFunction &MF) const
const TargetRegisterClass * getCrossCopyRegClass(const TargetRegisterClass *RC) const override
Returns a legal register class to copy a register in the specified class to or from.
ArrayRef< int16_t > getRegSplitParts(const TargetRegisterClass *RC, unsigned EltSize) const
ArrayRef< MCPhysReg > getAllSGPR32(const MachineFunction &MF) const
Return all SGPR32 which satisfy the waves per execution unit requirement of the subtarget.
const TargetRegisterClass * getLargestLegalSuperClass(const TargetRegisterClass *RC, const MachineFunction &MF) const override
MCRegister reservedPrivateSegmentBufferReg(const MachineFunction &MF) const
Return the end register initially reserved for the scratch buffer in case spilling is needed.
bool eliminateSGPRToVGPRSpillFrameIndex(MachineBasicBlock::iterator MI, int FI, RegScavenger *RS, SlotIndexes *Indexes=nullptr, LiveIntervals *LIS=nullptr, bool SpillToPhysVGPRLane=false) const
Special case of eliminateFrameIndex.
bool isVGPR(const MachineRegisterInfo &MRI, Register Reg) const
void buildSpillLoadStore(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, unsigned LoadStoreOp, int Index, Register ValueReg, bool ValueIsKill, MCRegister ScratchOffsetReg, int64_t InstrOffset, MachineMemOperand *MMO, RegScavenger *RS, LiveRegUnits *LiveUnits=nullptr) const
std::pair< unsigned, unsigned > getMaxNumVectorRegs(const MachineFunction &MF) const
Return a pair of maximum numbers of VGPRs and AGPRs that meet the number of waves per execution unit ...
bool isAsmClobberable(const MachineFunction &MF, MCRegister PhysReg) const override
LLVM_READONLY const TargetRegisterClass * getAGPRClassForBitWidth(unsigned BitWidth) const
static bool isChainScratchRegister(Register VGPR)
bool requiresRegisterScavenging(const MachineFunction &Fn) const override
bool opCanUseInlineConstant(unsigned OpType) const
const TargetRegisterClass * getRegClassForSizeOnBank(unsigned Size, const RegisterBank &Bank) const
const TargetRegisterClass * getConstrainedRegClassForOperand(const MachineOperand &MO, const MachineRegisterInfo &MRI) const override
bool isUniformReg(const MachineRegisterInfo &MRI, const RegisterBankInfo &RBI, Register Reg) const override
const uint32_t * getNoPreservedMask() const override
StringRef getRegAsmName(MCRegister Reg) const override
const uint32_t * getAllAllocatableSRegMask() const
MCRegister getAlignedHighSGPRForRC(const MachineFunction &MF, const unsigned Align, const TargetRegisterClass *RC) const
Return the largest available SGPR aligned to Align for the register class RC.
unsigned getNumUsedPhysRegs(const MachineRegisterInfo &MRI, const TargetRegisterClass &RC) const
const TargetRegisterClass * getRegClassForReg(const MachineRegisterInfo &MRI, Register Reg) const
const MCPhysReg * getCalleeSavedRegsViaCopy(const MachineFunction *MF) const
const uint32_t * getAllVectorRegMask() const
const TargetRegisterClass * getEquivalentAGPRClass(const TargetRegisterClass *SRC) const
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
const TargetRegisterClass * getRegClassForTypeOnBank(LLT Ty, const RegisterBank &Bank) const
bool opCanUseLiteralConstant(unsigned OpType) const
Register getBaseRegister() const
LLVM_READONLY const TargetRegisterClass * getVGPRClassForBitWidth(unsigned BitWidth) const
bool requiresFrameIndexScavenging(const MachineFunction &MF) const override
bool shouldRewriteCopySrc(const TargetRegisterClass *DefRC, unsigned DefSubReg, const TargetRegisterClass *SrcRC, unsigned SrcSubReg) const override
static bool isVGPRClass(const TargetRegisterClass *RC)
unsigned getHWRegIndex(MCRegister Reg) const
MachineInstr * findReachingDef(Register Reg, unsigned SubReg, MachineInstr &Use, MachineRegisterInfo &MRI, LiveIntervals *LIS) const
bool isSGPRReg(const MachineRegisterInfo &MRI, Register Reg) const
const TargetRegisterClass * getEquivalentSGPRClass(const TargetRegisterClass *VRC) const
SmallVector< StringLiteral > getVRegFlagsOfReg(Register Reg, const MachineFunction &MF) const override
unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const override
ArrayRef< MCPhysReg > getAllSGPR128(const MachineFunction &MF) const
Return all SGPR128 which satisfy the waves per execution unit requirement of the subtarget.
unsigned getRegPressureSetLimit(const MachineFunction &MF, unsigned Idx) const override
BitVector getReservedRegs(const MachineFunction &MF) const override
bool needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const override
const TargetRegisterClass * getRegClassForOperandReg(const MachineRegisterInfo &MRI, const MachineOperand &MO) const
const uint32_t * getAllAGPRRegMask() const
bool shouldCoalesce(MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg, const TargetRegisterClass *DstRC, unsigned DstSubReg, const TargetRegisterClass *NewRC, LiveIntervals &LIS) const override
const TargetRegisterClass * getBoolRC() const
const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind=0) const override
bool isAGPR(const MachineRegisterInfo &MRI, Register Reg) const
bool eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, RegScavenger *RS) const override
bool spillSGPR(MachineBasicBlock::iterator MI, int FI, RegScavenger *RS, SlotIndexes *Indexes=nullptr, LiveIntervals *LIS=nullptr, bool OnlyToVGPR=false, bool SpillToPhysVGPRLane=false) const
If OnlyToVGPR is true, this will only succeed if this manages to find a free VGPR lane to spill.
MCRegister getExec() const
MCRegister getVCC() const
int64_t getFrameIndexInstrOffset(const MachineInstr *MI, int Idx) const override
bool isVectorSuperClass(const TargetRegisterClass *RC) const
const TargetRegisterClass * getWaveMaskRegClass() const
unsigned getSubRegAlignmentNumBits(const TargetRegisterClass *RC, unsigned SubReg) const
void resolveFrameIndex(MachineInstr &MI, Register BaseReg, int64_t Offset) const override
bool requiresVirtualBaseRegisters(const MachineFunction &Fn) const override
const TargetRegisterClass * getVGPR64Class() const
void buildVGPRSpillLoadStore(SGPRSpillBuilder &SB, int Index, int Offset, bool IsLoad, bool IsKill=true) const
static bool isSGPRClass(const TargetRegisterClass *RC)
static bool isAGPRClass(const TargetRegisterClass *RC)
const int * getRegUnitPressureSets(unsigned RegUnit) const override
SlotIndex - An opaque wrapper around machine indexes.
bool isValid() const
Returns true if this is a valid index.
SlotIndex insertMachineInstrInMaps(MachineInstr &MI, bool Late=false)
Insert the given machine instruction into the mapping.
SlotIndex replaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
ReplaceMachineInstrInMaps - Replacing a machine instr with a new one in maps used by register allocat...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
const uint8_t TSFlags
Configurable target specific flags.
ArrayRef< MCPhysReg > getRegisters() const
unsigned getID() const
Return the register class ID number.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
virtual const TargetRegisterClass * getLargestLegalSuperClass(const TargetRegisterClass *RC, const MachineFunction &) const
Returns the largest super class of RC that is legal to use in the current sub-target and has the same...
virtual bool shouldRealignStack(const MachineFunction &MF) const
True if storage within the function requires the stack pointer to be aligned more than the normal cal...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
A Use represents the edge between a Value definition and its users.
VNInfo - Value Number Information.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ PRIVATE_ADDRESS
Address space for private memory.
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
LLVM_READONLY int getFlatScratchInstSVfromSS(uint16_t Opcode)
LLVM_READONLY int getFlatScratchInstSTfromSS(uint16_t Opcode)
LLVM_READONLY int getFlatScratchInstSVfromSVS(uint16_t Opcode)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
LLVM_READNONE bool isInlinableIntLiteral(int64_t Literal)
Is this literal inlinable, and not one of the values intended for floating point values.
@ OPERAND_REG_INLINE_AC_FIRST
@ OPERAND_REG_INLINE_AC_LAST
unsigned getRegBitWidth(const TargetRegisterClass &RC)
Get the size in bits of a register from the register class RC.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
auto reverse(ContainerTy &&C)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
void call_once(once_flag &flag, Function &&F, Args &&... ArgList)
Execute the function specified as a parameter once.
constexpr unsigned BitWidth
static const MachineMemOperand::Flags MOLastUse
Mark the MMO of a load as the last use.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
void setMI(MachineBasicBlock *NewMBB, MachineBasicBlock::iterator NewMI)
ArrayRef< int16_t > SplitParts
SIMachineFunctionInfo & MFI
SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, bool IsWave32, MachineBasicBlock::iterator MI, int Index, RegScavenger *RS)
SGPRSpillBuilder(const SIRegisterInfo &TRI, const SIInstrInfo &TII, bool IsWave32, MachineBasicBlock::iterator MI, Register Reg, bool IsKill, int Index, RegScavenger *RS)
PerVGPRData getPerVGPRData()
MachineBasicBlock::iterator MI
void readWriteTmpVGPR(unsigned Offset, bool IsLoad)
const SIRegisterInfo & TRI
The llvm::once_flag structure.