25#define DEBUG_TYPE "frame-info"
35 return ST.isWave64() && (
Reg == AMDGPU::VCC_LO ||
Reg == AMDGPU::VCC_HI);
57 OS <<
uint8_t(dwarf::DW_OP_reg0 + DwarfReg);
59 OS <<
uint8_t(dwarf::DW_OP_regx);
65 int64_t DwarfStackPtrReg) {
66 assert(ST.hasFlatScratchEnabled());
74 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
75 assert(WavefrontSizeLog2 < 32);
81 <<
uint8_t(dwarf::DW_OP_lit0 + WavefrontSizeLog2)
84 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave)
85 <<
uint8_t(dwarf::DW_OP_LLVM_user)
86 <<
uint8_t(dwarf::DW_OP_LLVM_form_aspace_address);
90 OSCFIInst <<
uint8_t(dwarf::DW_CFA_def_cfa_expression);
100 bool AspaceAlreadyDefined,
104 const SIRegisterInfo *
TRI =
ST.getRegisterInfo();
106 int64_t DwarfStackPtrReg =
TRI->getDwarfRegNum(StackPtrReg,
false);
107 MCCFIInstruction CFIInst =
108 ST.hasFlatScratchEnabled()
110 : (AspaceAlreadyDefined
111 ? MCCFIInstruction::createLLVMDefAspaceCfa(
112 nullptr, DwarfStackPtrReg, 0,
113 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave, SMLoc())
114 : MCCFIInstruction::createDefCfaRegister(nullptr,
128 for (
unsigned i = 0; CSRegs[i]; ++i)
129 LiveUnits.
addReg(CSRegs[i]);
149 bool IncludeScratchCopy =
true) {
155 unsigned Size =
TRI->getSpillSize(RC);
156 Align Alignment =
TRI->getSpillAlign(RC);
164 if (IncludeScratchCopy)
168 int FI = FrameInfo.CreateStackObject(
Size, Alignment,
true,
nullptr,
171 if (
TRI->spillSGPRToVGPR() &&
188 FI = FrameInfo.CreateSpillStackObject(
Size, Alignment);
199 LiveUnits.
addReg(ScratchSGPR);
214 int64_t DwordOff = 0) {
215 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
216 : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
222 FrameInfo.getObjectAlign(FI));
223 LiveUnits.
addReg(SpillReg);
224 bool IsKill = !
MBB.isLiveIn(SpillReg);
225 TRI.buildSpillLoadStore(
MBB,
I,
DL,
Opc, FI, SpillReg, IsKill, FrameReg,
226 DwordOff, MMO,
nullptr, &LiveUnits);
238 Register FrameReg, int64_t DwordOff = 0) {
239 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
240 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
246 FrameInfo.getObjectAlign(FI));
247 TRI.buildSpillLoadStore(
MBB,
I,
DL,
Opc, FI, SpillReg,
false, FrameReg,
248 DwordOff, MMO,
nullptr, &LiveUnits);
258 Register TargetLo =
TRI->getSubReg(TargetReg, AMDGPU::sub0);
259 Register TargetHi =
TRI->getSubReg(TargetReg, AMDGPU::sub1);
266 const MCInstrDesc &GetPC64 =
TII->get(AMDGPU::S_GETPC_B64_pseudo);
271 MBB.addLiveIn(GitPtrLo);
280 if (LiveUnits.
empty()) {
316 unsigned EltSize = 4;
317 bool IsFramePtrPrologSpill;
318 bool NeedsFrameMoves;
321 return Reg == AMDGPU::EXEC_LO || Reg == AMDGPU::EXEC;
333 if (IsFramePtrPrologSpill)
334 return FuncInfo->getFrameOffsetReg();
337 if (isExec(SuperReg))
342 void saveToMemory(
const int FI)
const {
345 assert(!MFI.isDeadObjectIndex(FI));
350 MRI, LiveUnits, AMDGPU::VGPR_32RegClass);
355 TFI->buildCFI(MBB, MI, DL,
357 nullptr, MCRI->getDwarfRegNum(Reg,
false),
358 MFI.getObjectOffset(FI) * ST.getWavefrontSize()));
361 for (
unsigned I = 0, DwordOff = 0;
I < NumSubRegs; ++
I) {
364 :
Register(TRI.getSubReg(SuperReg, SplitParts[
I]));
365 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
369 FI, FrameReg, DwordOff);
370 if (NeedsFrameMoves && !CFISuperReg)
374 if (NeedsFrameMoves && CFISuperReg)
375 BuildCFI(CFISuperReg);
378 void saveToVGPRLane(
const int FI)
const {
379 assert(!MFI.isDeadObjectIndex(FI));
383 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FI);
384 assert(Spill.size() == NumSubRegs);
387 for (
unsigned I = 0;
I < NumSubRegs; ++
I) {
390 :
Register(TRI.getSubReg(SuperReg, SplitParts[
I]));
391 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_S32_TO_VGPR),
396 if (NeedsFrameMoves && !CFISuperReg)
397 TFI->buildCFIForSGPRToVGPRSpill(MBB, MI, DL, SubReg, Spill[
I].VGPR,
400 if (NeedsFrameMoves && CFISuperReg)
401 TFI->buildCFIForSGPRToVGPRSpill(MBB, MI, DL, CFISuperReg, Spill);
404 void copyToScratchSGPR(
Register DstReg)
const {
405 BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), DstReg)
408 if (NeedsFrameMoves) {
411 CFISuperReg = SuperReg;
413 assert(NumSubRegs == 1 &&
"Expected a 32-bit SGPR");
414 TFI->buildCFIForSGPRToVCCHalfCopy(MBB, MI, DL, CFISuperReg, DstReg);
416 TFI->buildCFIForSRegToSRegSpill(MBB, MI, DL, CFISuperReg, DstReg);
421 void restoreFromMemory(
const int FI) {
427 MRI, LiveUnits, AMDGPU::VGPR_32RegClass);
431 for (
unsigned I = 0, DwordOff = 0;
I < NumSubRegs; ++
I) {
434 : TRI.getSubReg(SuperReg, SplitParts[
I]);
437 TmpVGPR, FI, FrameReg, DwordOff);
440 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg)
446 void restoreFromVGPRLane(
const int FI) {
449 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FI);
450 assert(Spill.size() == NumSubRegs);
452 for (
unsigned I = 0;
I < NumSubRegs; ++
I) {
455 : TRI.getSubReg(SuperReg, SplitParts[
I]);
456 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_RESTORE_S32_FROM_VGPR), SubReg)
462 void copyFromScratchSGPR(
Register SrcReg)
const {
463 BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SuperReg)
476 bool IsFramePtrPrologSpill =
false)
477 : MI(MI), MBB(MBB), MF(*MBB.
getParent()),
478 ST(MF.getSubtarget<
GCNSubtarget>()), MFI(MF.getFrameInfo()),
480 MCRI(MF.
getContext().getRegisterInfo()), TFI(ST.getFrameLowering()),
481 SuperReg(Reg), SI(SI), LiveUnits(LiveUnits), DL(DL), FrameReg(FrameReg),
482 IsFramePtrPrologSpill(IsFramePtrPrologSpill),
483 NeedsFrameMoves(MF.needsFrameMoves()) {
485 SplitParts = TRI.getRegSplitParts(RC, EltSize);
486 NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
488 assert(SuperReg != AMDGPU::M0 &&
"m0 should never spill");
492 switch (SI.getKind()) {
494 return saveToMemory(SI.getIndex());
496 return saveToVGPRLane(SI.getIndex());
498 return copyToScratchSGPR(SI.getReg());
503 switch (SI.getKind()) {
505 return restoreFromMemory(SI.getIndex());
507 return restoreFromVGPRLane(SI.getIndex());
509 return copyFromScratchSGPR(SI.getReg());
517void SIFrameLowering::emitEntryFunctionFlatScratchInit(
521 const SIInstrInfo *
TII =
ST.getInstrInfo();
523 const SIMachineFunctionInfo *MFI = MF.
getInfo<SIMachineFunctionInfo>();
538 if (
ST.isAmdPalOS()) {
540 LiveRegUnits LiveUnits;
546 Register FlatScrInit = AMDGPU::NoRegister;
549 AllSGPR64s = AllSGPR64s.
slice(
550 std::min(
static_cast<unsigned>(AllSGPR64s.
size()), NumPreloaded));
559 assert(FlatScrInit &&
"Failed to find free register for scratch init");
561 FlatScrInitLo =
TRI->getSubReg(FlatScrInit, AMDGPU::sub0);
562 FlatScrInitHi =
TRI->getSubReg(FlatScrInit, AMDGPU::sub1);
569 const MCInstrDesc &LoadDwordX2 =
TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
586 const MCInstrDesc &SAndB32 =
TII->get(AMDGPU::S_AND_B32);
594 assert(FlatScratchInitReg);
600 FlatScrInitLo =
TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0);
601 FlatScrInitHi =
TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1);
605 if (
ST.flatScratchIsPointer()) {
609 .
addReg(ScratchWaveOffsetReg);
616 using namespace AMDGPU::Hwreg;
619 .
addImm(int16_t(HwregEncoding::encode(ID_FLAT_SCR_LO, 0, 32)));
622 .
addImm(int16_t(HwregEncoding::encode(ID_FLAT_SCR_HI, 0, 32)));
629 .
addReg(ScratchWaveOffsetReg);
649 .
addReg(ScratchWaveOffsetReg);
672Register SIFrameLowering::getEntryFunctionReservedScratchRsrcReg(
676 const SIInstrInfo *
TII =
ST.getInstrInfo();
679 SIMachineFunctionInfo *MFI = MF.
getInfo<SIMachineFunctionInfo>();
689 if (
ST.hasSGPRInitBug() ||
690 ScratchRsrcReg !=
TRI->reservedPrivateSegmentBufferReg(MF))
691 return ScratchRsrcReg;
704 AllSGPR128s = AllSGPR128s.
slice(std::min(
static_cast<unsigned>(AllSGPR128s.
size()), NumPreloaded));
714 (!GITPtrLoReg || !
TRI->isSubRegisterEq(
Reg, GITPtrLoReg))) {
722 return ScratchRsrcReg;
726 return ST.hasFlatScratchEnabled() ? 1 : ST.getWavefrontSize();
731 assert(&MF.
front() == &
MBB &&
"Shrink-wrapping not yet supported");
762 static const char CFAEncodedInstUserOpsArr[] = {
763 dwarf::DW_CFA_def_cfa_expression,
765 static_cast<char>(dwarf::DW_OP_lit0),
766 static_cast<char>(dwarf::DW_OP_lit0 +
767 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave),
768 static_cast<char>(dwarf::DW_OP_LLVM_user),
769 static_cast<char>(dwarf::DW_OP_LLVM_form_aspace_address)};
771 StringRef(CFAEncodedInstUserOpsArr,
sizeof(CFAEncodedInstUserOpsArr));
775 "CFA is 0 in private_wave aspace"));
779 nullptr,
TRI->getDwarfRegNum(AMDGPU::PC_REG,
false)));
792 if (!ST.hasFlatScratchEnabled())
793 ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
796 if (ScratchRsrcReg) {
798 if (&OtherBB != &
MBB) {
799 OtherBB.addLiveIn(ScratchRsrcReg);
807 if (ST.isAmdHsaOrMesa(
F)) {
808 PreloadedScratchRsrcReg =
810 if (ScratchRsrcReg && PreloadedScratchRsrcReg) {
813 MRI.addLiveIn(PreloadedScratchRsrcReg);
814 MBB.addLiveIn(PreloadedScratchRsrcReg);
824 if (PreloadedScratchWaveOffsetReg &&
825 TRI->isSubRegisterEq(ScratchRsrcReg, PreloadedScratchWaveOffsetReg)) {
828 AllSGPRs = AllSGPRs.
slice(
829 std::min(
static_cast<unsigned>(AllSGPRs.
size()), NumPreloaded));
832 if (!
MRI.isPhysRegUsed(Reg) &&
MRI.isAllocatable(Reg) &&
833 !
TRI->isSubRegisterEq(ScratchRsrcReg, Reg) && GITPtrLoReg != Reg) {
834 ScratchWaveOffsetReg = Reg;
843 if (!ScratchWaveOffsetReg)
845 "could not find temporary scratch offset register in prolog");
847 ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
849 assert(ScratchWaveOffsetReg || !PreloadedScratchWaveOffsetReg);
879 FrameInfo.getMaxAlign());
891 ST.hasInv2PiInlineImm())) {
903 bool NeedsFlatScratchInit =
905 (
MRI.isPhysRegUsed(AMDGPU::FLAT_SCR) || FrameInfo.hasCalls() ||
908 if ((NeedsFlatScratchInit || ScratchRsrcReg) &&
909 PreloadedScratchWaveOffsetReg && !ST.hasArchitectedFlatScratch()) {
910 MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
911 MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
914 if (NeedsFlatScratchInit) {
915 emitEntryFunctionFlatScratchInit(MF,
MBB,
I,
DL, ScratchWaveOffsetReg);
918 if (ScratchRsrcReg) {
919 emitEntryFunctionScratchRsrcRegSetup(MF,
MBB,
I,
DL,
920 PreloadedScratchRsrcReg,
921 ScratchRsrcReg, ScratchWaveOffsetReg);
935 assert(
MBBI !=
MBB.end() &&
"return block should end in a terminator");
970void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
981 if (ST.isAmdPalOS()) {
984 Register Rsrc01 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
985 Register Rsrc03 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
992 const MCInstrDesc &LoadDwordX4 =
TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
1015 if (ST.isWave32()) {
1021 }
else if (
ST.isMesaGfxShader(Fn) || !PreloadedScratchRsrcReg) {
1023 const MCInstrDesc &SMovB32 =
TII->get(AMDGPU::S_MOV_B32);
1025 Register Rsrc2 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
1026 Register Rsrc3 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
1032 Register Rsrc01 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
1035 const MCInstrDesc &Mov64 =
TII->get(AMDGPU::S_MOV_B64);
1041 const MCInstrDesc &LoadDwordX2 =
TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
1060 Register Rsrc0 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
1061 Register Rsrc1 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
1079 }
else if (
ST.isAmdHsaOrMesa(Fn)) {
1080 assert(PreloadedScratchRsrcReg);
1082 if (ScratchRsrcReg != PreloadedScratchRsrcReg) {
1097 Register ScratchRsrcSub0 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
1098 Register ScratchRsrcSub1 =
TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
1104 .
addReg(ScratchWaveOffsetReg)
1106 auto Addc =
BuildMI(
MBB,
I,
DL,
TII->get(AMDGPU::S_ADDC_U32), ScratchRsrcSub1)
1139 emitDefCFA(
MBB,
MBBI,
DL, StackPtrReg,
true,
1143 TRI.getReturnAddressReg(MF));
1147 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
1148 IsCalleeSaved.set(CSRegs[
I]);
1152 if (
Reg == AMDGPU::VCC ||
Reg == AMDGPU::VCC_LO ||
Reg == AMDGPU::VCC_HI)
1154 if (IsCalleeSaved.test(
Reg) || !
MRI.isPhysRegModified(
Reg))
1162 unsigned NumArchVGPRs =
ST.getAddressableNumArchVGPRs();
1163 for_each(AMDGPU::VGPR_32RegClass.getRegisters().take_front(NumArchVGPRs),
1167 if (
ST.hasMAIInsts()) {
1168 for_each(AMDGPU::AGPR_32RegClass.getRegisters(), ProcessReg);
1172 for_each(AMDGPU::SGPR_32RegClass.getRegisters(), ProcessReg);
1182 bool EnableInactiveLanes) {
1195 assert(IsProlog &&
"Epilog should look at return, not setup");
1197 TII->getWholeWaveFunctionSetup(MF)->getOperand(0).getReg();
1198 assert(ScratchExecCopy &&
"Couldn't find copy of EXEC");
1201 MRI, LiveUnits, *
TRI.getWaveMaskRegClass());
1204 if (!ScratchExecCopy)
1207 LiveUnits.
addReg(ScratchExecCopy);
1209 const unsigned SaveExecOpc =
1210 ST.isWave32() ? (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B32
1211 : AMDGPU::S_OR_SAVEEXEC_B32)
1212 : (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B64
1213 : AMDGPU::S_OR_SAVEEXEC_B64);
1218 return ScratchExecCopy;
1225 const bool NeedsFrameMoves)
const {
1241 if (!WWMScratchRegs.
empty())
1246 auto StoreWWMRegisters =
1248 for (
const auto &Reg : WWMRegs) {
1250 int FI = Reg.second;
1252 VGPR, FI, FrameReg);
1253 if (NeedsFrameMoves) {
1258 MFI.getObjectOffset(FI) * ST.getWavefrontSize()));
1264 if (!
MRI.isReserved(Reg)) {
1269 StoreWWMRegisters(WWMScratchRegs);
1271 auto EnableAllLanes = [&]() {
1275 if (!WWMCalleeSavedRegs.
empty()) {
1276 if (ScratchExecCopy) {
1285 StoreWWMRegisters(WWMCalleeSavedRegs);
1290 if (!ScratchExecCopy)
1293 else if (WWMCalleeSavedRegs.
empty())
1295 }
else if (ScratchExecCopy) {
1299 LiveUnits.
addReg(ScratchExecCopy);
1309 bool IsFramePtrPrologSpill = Spill.first == FramePtrReg;
1310 Register Reg = IsFramePtrPrologSpill ? FramePtrRegScratchCopy : Spill.first;
1315 LiveUnits, FrameReg, IsFramePtrPrologSpill);
1323 if (!ScratchSGPRs.
empty()) {
1328 MBB.sortUniqueLiveIns();
1330 if (!LiveUnits.
empty()) {
1338 if (
TRI.isCFISavedRegsSpillEnabled())
1346 Register FramePtrRegScratchCopy)
const {
1361 Spill.first == FramePtrReg ? FramePtrRegScratchCopy : Spill.first;
1366 LiveUnits, FrameReg);
1376 auto RestoreWWMRegisters =
1378 for (
const auto &Reg : WWMRegs) {
1380 int FI = Reg.second;
1382 VGPR, FI, FrameReg);
1389 RestoreWWMRegisters(WWMCalleeSavedRegs);
1393 unsigned Opcode = Return.getOpcode();
1395 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
1396 Opcode = AMDGPU::SI_RETURN;
1398 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
1399 Opcode = AMDGPU::SI_TCRETURN_GFX;
1404 Register OrigExec = Return.getOperand(0).getReg();
1406 if (!WWMScratchRegs.
empty()) {
1410 RestoreWWMRegisters(WWMScratchRegs);
1417 Return.removeOperand(0);
1418 Return.setDesc(
TII->get(Opcode));
1423 if (!WWMScratchRegs.
empty()) {
1428 RestoreWWMRegisters(WWMScratchRegs);
1429 if (!WWMCalleeSavedRegs.
empty()) {
1430 if (ScratchExecCopy) {
1439 RestoreWWMRegisters(WWMCalleeSavedRegs);
1440 if (ScratchExecCopy) {
1475 int64_t RoundedSize = NumBytes;
1479 bool SavesStackRegs =
1484 if (NeedsFrameMoves)
1487 if (
TRI.hasStackRealignment(MF))
1491 if (!HasFP && !
hasFP(MF)) {
1494 FramePtrRegScratchCopy, NeedsFrameMoves);
1495 }
else if (SavesStackRegs) {
1497 Register SGPRForFPSaveRestoreCopy =
1501 if (SGPRForFPSaveRestoreCopy) {
1508 DL,
TII,
TRI, LiveUnits, FramePtrReg,
1511 LiveUnits.
addReg(SGPRForFPSaveRestoreCopy);
1516 MRI, LiveUnits, AMDGPU::SReg_32_XM0_XEXECRegClass);
1517 if (!FramePtrRegScratchCopy)
1520 LiveUnits.
addReg(FramePtrRegScratchCopy);
1529 RoundedSize += Alignment;
1530 if (LiveUnits.
empty()) {
1545 And->getOperand(3).setIsDead();
1547 }
else if ((HasFP =
hasFP(MF))) {
1556 FramePtrRegScratchCopy, NeedsFrameMoves);
1557 if (FramePtrRegScratchCopy)
1558 LiveUnits.
removeReg(FramePtrRegScratchCopy);
1565 if ((HasBP =
TRI.hasBasePointer(MF))) {
1572 if (NeedsFrameMoves)
1573 emitDefCFA(
MBB,
MBBI,
DL, FramePtrReg,
false,
1577 if (HasFP && RoundedSize != 0) {
1582 Add->getOperand(3).setIsDead();
1587 assert((!HasFP || FPSaved || !SavesStackRegs) &&
1588 "Needed to save FP but didn't save it anywhere");
1592 assert((HasFP || !FPSaved || !SavesStackRegs ||
1593 ST.getCLOpts().spill_vgpr_to_agpr) &&
1594 "Saved FP but didn't need it");
1598 assert((!HasBP || BPSaved || !SavesStackRegs) &&
1599 "Needed to save BP but didn't save it anywhere");
1601 assert((HasBP || !BPSaved) &&
"Saved BP but didn't need it");
1605 TII->getWholeWaveFunctionSetup(MF)->eraseFromParent();
1631 MBBI =
MBB.getLastNonDebugInstr();
1633 DL =
MBBI->getDebugLoc();
1635 MBBI =
MBB.getFirstTerminator();
1646 if (RoundedSize != 0) {
1647 if (
TRI.hasBasePointer(MF)) {
1651 }
else if (
hasFP(MF)) {
1659 Register SGPRForFPSaveRestoreCopy =
1667 if (SGPRForFPSaveRestoreCopy) {
1668 LiveUnits.
addReg(SGPRForFPSaveRestoreCopy);
1671 MRI, LiveUnits, AMDGPU::SReg_32_XM0_XEXECRegClass);
1672 if (!FramePtrRegScratchCopy)
1675 LiveUnits.
addReg(FramePtrRegScratchCopy);
1679 FramePtrRegScratchCopy);
1683 emitDefCFA(
MBB,
MBBI,
DL, StackPtrReg,
false,
1689 Register SrcReg = SGPRForFPSaveRestoreCopy ? SGPRForFPSaveRestoreCopy
1690 : FramePtrRegScratchCopy;
1694 if (SGPRForFPSaveRestoreCopy)
1699 FramePtrRegScratchCopy);
1740 const bool SpillVGPRToAGPR = ST.hasMAIInsts() &&
1742 ST.getCLOpts().spill_vgpr_to_agpr;
1744 if (SpillVGPRToAGPR) {
1749 bool SeenDbgInstr =
false;
1754 if (
MI.isDebugInstr())
1755 SeenDbgInstr =
true;
1757 if (
TII->isVGPRSpill(
MI)) {
1760 unsigned FIOp = AMDGPU::getNamedOperandIdx(
MI.getOpcode(),
1761 AMDGPU::OpName::vaddr);
1762 int FI =
MI.getOperand(FIOp).getIndex();
1764 TII->getNamedOperand(
MI, AMDGPU::OpName::vdata)->getReg();
1766 TRI->isAGPR(
MRI, VReg))) {
1768 RS->enterBasicBlockEnd(
MBB);
1769 RS->backward(std::next(
MI.getIterator()));
1770 TRI->eliminateFrameIndex(
MI, 0, FIOp, RS);
1774 }
else if (
TII->isStoreToStackSlot(
MI, FrameIndex) ||
1775 TII->isLoadFromStackSlot(
MI, FrameIndex))
1777 NonVGPRSpillFIs.
set(FrameIndex);
1783 for (
unsigned FI : SpillFIs.
set_bits())
1784 if (!NonVGPRSpillFIs.
test(FI))
1794 MBB.sortUniqueLiveIns();
1796 if (!SpillFIs.
empty() && SeenDbgInstr)
1804 bool HaveSGPRToVMemSpill =
1807 "SGPR spill should have been removed in SILowerSGPRSpills");
1813 assert(RS &&
"RegScavenger required if spilling");
1824 }
else if (HaveSGPRToVMemSpill &&
1840 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
1847 TRI->findUnusedRegister(
MRI, &AMDGPU::VGPR_32RegClass, MF);
1848 if (UnusedLowVGPR && (
TRI->getHWRegIndex(UnusedLowVGPR) <
1849 TRI->getHWRegIndex(VGPRForAGPRCopy))) {
1855 MRI.reserveReg(UnusedLowVGPR,
TRI);
1862 TRI->findUnusedRegister(
MRI, &AMDGPU::SGPR_64RegClass, MF);
1867 if (LongBranchReservedReg && UnusedLowSGPR) {
1869 MRI.reserveReg(UnusedLowSGPR,
TRI);
1877 bool NeedExecCopyReservedReg)
const {
1888 for (
unsigned I = 0; CSRegs[
I]; ++
I)
1894 if (NeedExecCopyReservedReg ||
1895 (ReservedRegForExecCopy &&
1896 MRI.isPhysRegUsed(ReservedRegForExecCopy,
true))) {
1897 MRI.reserveReg(ReservedRegForExecCopy,
TRI);
1899 if (UnusedScratchReg) {
1903 MRI.replaceRegWith(ReservedRegForExecCopy, UnusedScratchReg);
1904 LiveUnits.
addReg(UnusedScratchReg);
1908 "Re-reserving spill slot for EXEC copy register");
1912 }
else if (ReservedRegForExecCopy) {
1918 if (
TRI->isCFISavedRegsSpillEnabled()) {
1921 "Re-reserving spill slot for EXEC");
1928 if (
F.hasFnAttribute(Attribute::NoReturn) ||
1940 const bool WillHaveFP =
1941 FrameInfo.hasCalls() &&
1944 if (WillHaveFP ||
hasFP(MF)) {
1947 "Re-reserving spill slot for FP");
1951 if (
TRI->hasBasePointer(MF)) {
1954 "Re-reserving spill slot for BP");
1976 bool NeedExecCopyReservedReg =
false;
1983 if (
TII->isWWMRegSpillOpcode(
MI.getOpcode()))
1984 NeedExecCopyReservedReg =
true;
1985 else if (
MI.getOpcode() == AMDGPU::SI_RETURN ||
1986 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1987 MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
1989 TII->isChainCallOpcode(
MI.getOpcode()))) {
1992 (
count_if(
MI.operands(), [](
auto Op) { return Op.isReg(); }) ==
2005 if (
TRI->getRegSizeInBits(*RC) != 32)
2010 sort(SortedWWMVGPRs, std::greater<Register>());
2019 assert(!NeedExecCopyReservedReg &&
2020 "Whole wave functions can use the reg mapped for their i1 argument");
2022 unsigned NumArchVGPRs = ST.getAddressableNumArchVGPRs();
2024 AMDGPU::VGPR_32RegClass.getRegisters().take_front(NumArchVGPRs))
2027 MF.
begin()->addLiveIn(Reg);
2029 MF.
begin()->sortUniqueLiveIns();
2037 SavedVGPRs.
reset(
Op.getReg());
2045 TRI->getSpillAlign(*RC));
2054 if (!ST.hasGFX90AInsts())
2062 SavedVGPRs.
reset(Reg.first);
2079 const BitVector AllSavedRegs = SavedRegs;
2088 const bool WillHaveFP =
2092 if (WillHaveFP ||
hasFP(MF))
2102 Register RetAddrReg =
TRI->getReturnAddressReg(MF);
2104 (FrameInfo.hasCalls() ||
MRI.isPhysRegModified(RetAddrReg))) {
2105 SavedRegs.
set(
TRI->getSubReg(RetAddrReg, AMDGPU::sub0));
2106 SavedRegs.
set(
TRI->getSubReg(RetAddrReg, AMDGPU::sub1));
2112 std::vector<CalleeSavedInfo> &CSI) {
2120 return A.getReg() <
B.getReg();
2122 "Callee saved registers not sorted");
2125 return !CSI.isSpilledToReg() &&
2126 TRI->getPhysRegBaseClass(CSI.getReg()) == &AMDGPU::VGPR_32RegClass &&
2130 auto CSEnd = CSI.end();
2131 for (
auto CSIt = CSI.begin(); CSIt != CSEnd; ++CSIt) {
2133 if (!CanUseBlockOps(*CSIt))
2140 CSEnd = std::remove_if(
2142 if (CanUseBlockOps(CSI) && CSI.
getReg() <
Reg + 32) {
2152 TRI->getMatchingSuperReg(
Reg, AMDGPU::sub0, BlockRegClass);
2161 TRI->getMatchingSuperReg(LastBlockStart, AMDGPU::sub0, BlockRegClass);
2162 assert(RegBlock &&
TRI->isSubRegister(RegBlock,
Reg) &&
2163 "Couldn't find super register");
2164 int RegDelta =
Reg - LastBlockStart;
2166 "Bad shift amount");
2177 unsigned BlockSize =
TRI->getSpillSize(*BlockRegClass) - UnusedBits * 4;
2179 MFI.CreateStackObject(
BlockSize,
TRI->getSpillAlign(*BlockRegClass),
2181 MFI.setIsCalleeSavedObjectIndex(FrameIdx,
true);
2183 CSIt->setFrameIdx(FrameIdx);
2184 CSIt->setReg(RegBlock);
2186 CSI.erase(CSEnd, CSI.end());
2191 std::vector<CalleeSavedInfo> &CSI)
const {
2196 bool UseVGPRBlocks = ST.useVGPRBlockOpsForCSR();
2206 std::vector<CalleeSavedInfo> &CSI)
const {
2214 Register BasePtrReg = RI->getBaseRegister();
2215 Register SGPRForFPSaveRestoreCopy =
2217 Register SGPRForBPSaveRestoreCopy =
2219 if (!SGPRForFPSaveRestoreCopy && !SGPRForBPSaveRestoreCopy)
2222 unsigned NumModifiedRegs = 0;
2224 if (SGPRForFPSaveRestoreCopy)
2226 if (SGPRForBPSaveRestoreCopy)
2229 for (
auto &CS : CSI) {
2230 if (CS.getReg() == FramePtrReg.
asMCReg() && SGPRForFPSaveRestoreCopy) {
2231 CS.setDstReg(SGPRForFPSaveRestoreCopy);
2232 if (--NumModifiedRegs)
2234 }
else if (CS.getReg() == BasePtrReg.
asMCReg() &&
2235 SGPRForBPSaveRestoreCopy) {
2236 CS.setDstReg(SGPRForBPSaveRestoreCopy);
2237 if (--NumModifiedRegs)
2251 uint64_t EstStackSize = MFI.estimateStackSize(MF);
2252 uint64_t MaxOffset = EstStackSize - 1;
2261 if (ST.hasFlatScratchEnabled()) {
2266 if (
TII->isLegalMUBUFImmOffset(MaxOffset))
2280 for (
const auto &LI :
MBB.liveins()) {
2282 auto [Unit, UnitLaneMask] = *
MI;
2283 if ((LI.LaneMask & UnitLaneMask).none())
2286 LiveInRoots.
set(*RI);
2298 if (LiveInRoots.
test(*RI))
2305void SIFrameLowering::spillCalleeSavedRegisterWithoutBlockOps(
2339 std::optional<SparseBitVector<>> LiveInRoots;
2340 if (
MBB.getParent()->getRegInfo().tracksLiveness())
2343 if (!ST.useVGPRBlockOpsForCSR()) {
2345 spillCalleeSavedRegisterWithoutBlockOps(
MBB,
MI, CS,
TII,
TRI,
2348 MBB.sortUniqueLiveIns();
2358 if (!BlockRegClass->contains(Reg) ||
2360 spillCalleeSavedRegisterWithoutBlockOps(
MBB,
MI, CS,
TII,
TRI,
2372 FrameInfo.getObjectSize(FrameIndex),
2373 FrameInfo.getObjectAlign(FrameIndex));
2376 TII->get(AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE))
2394 MBB.sortUniqueLiveIns();
2406 if (!ST.useVGPRBlockOpsForCSR())
2415 if (!BlockRegClass->
contains(Reg) ||
2428 MFI.getObjectAlign(FrameIndex));
2431 TII->get(AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE), Reg)
2437 TRI.addImplicitUsesForBlockCSRLoad(MIB, Reg);
2446 MBB.sortUniqueLiveIns();
2454 int64_t Amount =
I->getOperand(0).getImm();
2456 return MBB.erase(
I);
2461 unsigned Opc =
I->getOpcode();
2462 bool IsDestroy =
Opc ==
TII->getCallFrameDestroyOpcode();
2463 uint64_t CalleePopAmount = IsDestroy ?
I->getOperand(1).getImm() : 0;
2477 Add->getOperand(3).setIsDead();
2478 }
else if (CalleePopAmount != 0) {
2482 return MBB.erase(
I);
2536 "only expected to call this for entry points functions");
2570 ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC,
false);
2574 ST.getWavefrontSize());
2581 const int Lane)
const {
2587 assert(DwarfSGPR != -1 && DwarfVGPR != -1);
2588 assert(Lane != -1 &&
"Expected a lane to be present");
2603 if (VGPRSpills.
size() == 1u)
2605 VGPRSpills[0].Lane);
2619 assert(Spill.hasLane() &&
"Expected a lane to be present");
2650 ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC,
false);
2659 int64_t DwarfReg =
TRI.getDwarfRegNum(
Reg,
false);
2660 if (DwarfReg >= 0) {
2661 DwarfRegs.
push_back(
unsigned(DwarfReg));
2667 for (
unsigned I = 0;
I < SplitParts.
size(); ++
I) {
2669 DwarfReg =
TRI.getDwarfRegNum(SubReg,
false);
2674 DwarfRegs.
push_back(
unsigned(DwarfReg));
2689 if (DwarfRegs.
size() == DwarfCopyRegs.
size()) {
2690 for (
unsigned I = 0, E = DwarfRegs.
size();
I < E; ++
I) {
2698 if (DwarfRegs.
size() == 1 && DwarfCopyRegs.
size() == 2) {
2700 nullptr, DwarfRegs[0], DwarfCopyRegs[0],
SGPRBitSize, DwarfCopyRegs[1],
2706 for (
unsigned I = 0, E = DwarfRegs.
size();
I < E; ++
I) {
2721 assert(DwarfSGPR != -1 && DwarfVCC != -1);
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Provides AMDGPU specific target descriptions.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains constants used for implementing Dwarf debug support.
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static constexpr MCPhysReg FPReg
static constexpr MCPhysReg SPReg
MachineRegisterInfo & MRI
This file declares the machine register scavenger class.
static void buildEpilogRestore(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SpillReg, int FI, Register FrameReg, int64_t DwordOff=0)
static void getVGPRSpillLaneOrTempRegister(MachineFunction &MF, LiveRegUnits &LiveUnits, Register SGPR, const TargetRegisterClass &RC=AMDGPU::SReg_32_XM0_XEXECRegClass, bool IncludeScratchCopy=true)
Query target location for spilling SGPRs IncludeScratchCopy : Also look for free scratch SGPRs.
static void buildGitPtr(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, const SIInstrInfo *TII, Register TargetReg)
static bool allStackObjectsAreDead(const MachineFrameInfo &MFI)
static MCCFIInstruction createScaledCFAInPrivateWave(const GCNSubtarget &ST, int64_t DwarfStackPtrReg)
static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SpillReg, int FI, Register FrameReg, int64_t DwordOff=0)
static Register buildScratchExecCopy(LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool IsProlog, bool EnableInactiveLanes)
static void encodeDwarfRegisterLocation(int DwarfReg, raw_ostream &OS)
static constexpr unsigned SGPRBitSize
static bool frameTriviallyRequiresSP(const MachineFrameInfo &MFI)
Returns true if the frame will require a reference to the stack pointer.
static void decomposeDwarfRegs(const SIRegisterInfo &TRI, MCRegister Reg, SmallVectorImpl< unsigned > &DwarfRegs)
static SparseBitVector buildLiveInRoots(const MachineBasicBlock &MBB, const SIRegisterInfo &TRI)
Return the set of all root registers of regunits live-in to MBB.
static void initLiveUnits(LiveRegUnits &LiveUnits, const SIRegisterInfo &TRI, const SIMachineFunctionInfo *FuncInfo, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsProlog)
static constexpr unsigned VGPRLaneBitSize
static bool allSGPRSpillsAreDead(const MachineFunction &MF)
static bool isWave64VCCHalf(const GCNSubtarget &ST, MCRegister Reg)
static MCRegister findScratchNonCalleeSaveRegister(MachineRegisterInfo &MRI, LiveRegUnits &LiveUnits, const TargetRegisterClass &RC, bool Unused=false)
static MCRegister findUnusedRegister(MachineRegisterInfo &MRI, const LiveRegUnits &LiveUnits, const TargetRegisterClass &RC)
static constexpr unsigned SGPRByteSize
static void assignSlotsUsingVGPRBlocks(MachineFunction &MF, const GCNSubtarget &ST, std::vector< CalleeSavedInfo > &CSI)
static bool isAnyRootLiveIn(const SparseBitVector<> &LiveInRoots, const SIRegisterInfo &TRI, MCRegister Reg)
Returns true iff any root of Reg is in LiveInRoots (see buildLiveInRoots).
static unsigned getScratchScaleFactor(const GCNSubtarget &ST)
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static const int BlockSize
bool isChainFunction() const
bool isEntryFunction() const
static const LaneMaskConstants & get(const GCNSubtarget &ST)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
bool test(unsigned Idx) const
Returns true if bit Idx is set.
BitVector & reset()
Reset all bits in the bitvector.
void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
Clear a bit in this vector for every '0' bit in Mask.
BitVector & set()
Set all bits in the bitvector.
bool any() const
Returns true if any bit is set.
void clearBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
Clear any bits in this vector that are set in Mask.
iterator_range< const_set_bits_iterator > set_bits() const
bool empty() const
Returns whether there are no bits in this bitvector.
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
bool isSpilledToReg() const
MCRegister getReg() const
MCRegister getDstReg() const
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
bool hasImplicitBufferPtr() const
bool hasFlatScratchInit() const
const HexagonRegisterInfo & getRegisterInfo() const
A set of register units used to track register liveness.
bool available(MCRegister Reg) const
Returns true if no part of physical register Reg is live.
void init(const TargetRegisterInfo &TRI)
Initialize and clear the set.
void addReg(MCRegister Reg)
Adds register units covered by physical register Reg.
LLVM_ABI void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
LLVM_ABI void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
void removeReg(MCRegister Reg)
Removes all register units covered by physical register Reg.
bool empty() const
Returns true if the set is empty.
LLVM_ABI void addLiveIns(const MachineBasicBlock &MBB)
Adds registers living into block MBB.
static MCCFIInstruction createLLVMVectorOffset(MCSymbol *L, unsigned Register, unsigned RegisterSizeInBits, unsigned MaskRegister, unsigned MaskRegisterSizeInBits, int64_t Offset, SMLoc Loc={})
.cfi_llvm_vector_offset Previous value of Register is saved at Offset from CFA.
static MCCFIInstruction createUndefined(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_undefined From now on the previous value of Register can't be restored anymore.
static MCCFIInstruction createLLVMVectorRegisters(MCSymbol *L, unsigned Register, ArrayRef< VectorRegisterWithLane > VectorRegisters, SMLoc Loc={})
.cfi_llvm_vector_registers Previous value of Register is saved in lanes of vector registers.
static MCCFIInstruction createLLVMVectorRegisterMask(MCSymbol *L, unsigned Register, unsigned SpillRegister, unsigned SpillRegisterLaneSizeInBits, unsigned MaskRegister, unsigned MaskRegisterSizeInBits, SMLoc Loc={})
.cfi_llvm_vector_register_mask Previous value of Register is saved in SpillRegister,...
static MCCFIInstruction createRegister(MCSymbol *L, unsigned Register1, unsigned Register2, SMLoc Loc={})
.cfi_register Previous value of Register1 is saved in register Register2.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction createLLVMRegisterPair(MCSymbol *L, unsigned Register, unsigned R1, unsigned R1SizeInBits, unsigned R2, unsigned R2SizeInBits, SMLoc Loc={})
.cfi_llvm_register_pair Previous value of Register is saved in R1:R2.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
static MCCFIInstruction createSameValue(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_same_value Current value of Register is the same as in the previous frame.
const MCRegisterInfo * getRegisterInfo() const
Describe properties that are true of each instruction in the target description file.
bool isValid() const
Returns true if this iterator is not yet at the end.
MCRegUnitMaskIterator enumerates a list of register units and their associated lane masks for Reg.
MCRegUnitRootIterator enumerates the root registers of a register unit.
bool isValid() const
Check if the iterator is at the end of the list.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
virtual int64_t getDwarfRegNum(MCRegister Reg, bool isEH) const
Map a target register to an equivalent dwarf register number.
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
bool hasCalls() const
Return true if the current function has any function calls.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Align getMaxAlign() const
Return alignment of this function's frame.
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
bool hasTailCall() const
Returns true if the function contains a tail call.
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
LLVM_ABI int CreateSpillStackObject(uint64_t Size, Align Alignment, TargetStackID::Value StackID=TargetStackID::Default)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getObjectIndexBegin() const
Return the minimum frame object index.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
bool needsFrameMoves() const
True if this function needs frame moves for debug or exceptions.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
bool disableFramePointerElim() const
Returns true if frame pointer elimination should be disabled for this function.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
void setIsDead(bool Val=true)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool isReserved(MCRegister PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
bool isAllocatable(MCRegister PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn't been...
LLVM_ABI const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
void reserveReg(MCRegister PhysReg, const TargetRegisterInfo *TRI)
reserveReg – Mark a register as reserved so checks like isAllocatable will not suggest using it.
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
LLVM_ABI bool isPhysRegModified(MCRegister PhysReg, bool SkipNoReturnDef=false) const
Return true if the specified register is modified in this function.
LLVM_ABI bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
PrologEpilogSGPRSpillBuilder(Register Reg, const PrologEpilogSGPRSaveRestoreInfo SI, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, const SIInstrInfo *TII, const SIRegisterInfo &TRI, LiveRegUnits &LiveUnits, Register FrameReg, bool IsFramePtrPrologSpill=false)
Wrapper class representing virtual and physical registers.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
void determinePrologEpilogSGPRSaves(MachineFunction &MF, BitVector &SavedRegs, bool NeedExecCopyReservedReg) const
MachineInstr * buildCFIForSGPRToVMEMSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister SGPR, int64_t Offset) const
Create a CFI index describing a spill of a SGPR to VMEM and build a MachineInstr around it.
void buildCFIForSRegToSRegSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister Reg, MCRegister CopyReg) const
void emitCSRSpillRestores(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy) const
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - This method should return the base register and offset used to reference a f...
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS=nullptr) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
bool mayReserveScratchForCWSR(const MachineFunction &MF) const
bool allocateScavengingFrameIndexesNearIncomingSP(const MachineFunction &MF) const override
Control the placement of special register scavenging spill slots when allocating a stack frame.
bool requiresStackPointerReference(const MachineFunction &MF) const
void emitEntryFunctionPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const
MachineInstr * buildCFIForSGPRToVCCHalfCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister SGPR, MCRegister VCCHalf) const
Create a CFI index describing a copy of the 32-bit SGPR to VCCHalf (vcc_lo or vcc_hi) in wave64 and b...
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
bool hasFPImpl(const MachineFunction &MF) const override
bool assignCalleeSavedSpillSlotsImpl(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI) const
MachineInstr * buildCFIForVRegToVRegSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCRegister Reg, const MCRegister RegCopy) const
Create a CFI index describing a spill of the VGPR/AGPR Reg to another VGPR/AGPR RegCopy and build a M...
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
MachineInstr * buildCFIForVGPRToVMEMSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister VGPR, int64_t Offset) const
Create a CFI index describing a spill of a VGPR to VMEM and build a MachineInstr around it.
void emitEntryFunctionEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const
MachineInstr * buildCFIForSGPRToVGPRSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCRegister SGPR, const MCRegister VGPR, const int Lane) const
Create a CFI index describing a spill of an SGPR to a single lane of a VGPR and build a MachineInstr ...
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
void determineCalleeSavesSGPR(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
MachineInstr * buildCFIForSameValue(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister Reg) const
MachineInstr * buildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCCFIInstruction &CFIInst, MachineInstr::MIFlag flag=MachineInstr::FrameSetup) const
Create a CFI index for CFIInst and build a MachineInstr around it.
void emitCSRSpillStores(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy, const bool NeedsFrameMoves) const
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS=nullptr) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
bool isSupportedStackID(TargetStackID::Value ID) const override
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
ArrayRef< PrologEpilogSGPRSpill > getPrologEpilogSGPRSpills() const
const WWMSpillsMap & getWWMSpills() const
void getAllScratchSGPRCopyDstRegs(SmallVectorImpl< Register > &Regs) const
ArrayRef< MCPhysReg > getAGPRSpillVGPRs() const
void setSGPRForEXECCopy(Register Reg)
void removePrologEpilogSGPRSpillEntry(Register Reg)
unsigned getNumPreloadedSGPRs() const
void shiftWwmVGPRsToLowestRange(MachineFunction &MF, SmallVectorImpl< Register > &WWMVGPRs, BitVector &SavedVGPRs)
void setMaskForVGPRBlockOps(Register RegisterBlock, uint32_t Mask)
GCNUserSGPRUsageInfo & getUserSGPRInfo()
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
Register getLongBranchReservedReg() const
unsigned getDynamicVGPRBlockSize() const
bool hasSpilledVGPRs() const
void setVGPRToAGPRSpillDead(int FrameIndex)
bool isWholeWaveFunction() const
Register getStackPtrOffsetReg() const
bool isStackRealigned() const
Register getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses.
ArrayRef< MCPhysReg > getVGPRSpillAGPRs() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
uint32_t getMaskForVGPRBlockOps(Register RegisterBlock) const
bool hasMaskForVGPRBlockOps(Register RegisterBlock) const
bool hasPrologEpilogSGPRSpillEntry(Register Reg) const
Register getGITPtrLoReg(const MachineFunction &MF) const
void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy)
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
bool hasNoWWMPoolSGPRSpillFallback() const
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
Register getSGPRForEXECCopy() const
bool isWWMReservedRegister(Register Reg) const
ArrayRef< SIRegisterInfo::SpilledReg > getSGPRSpillToPhysicalVGPRLanes(int FrameIndex) const
Register getVGPRForAGPRCopy() const
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool SpillToPhysVGPRLane=false, bool IsPrologEpilog=false)
Register getFrameOffsetReg() const
void setLongBranchReservedReg(Register Reg)
void setHasSpilledVGPRs(bool Spill=true)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
void setScratchReservedForDynamicVGPRs(unsigned SizeInBytes)
MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
const ReservedRegSet & getWWMReservedRegs() const
Register getImplicitBufferPtrUserSGPR() const
const PrologEpilogSGPRSaveRestoreInfo & getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const
void setIsStackRealigned(bool Realigned=true)
unsigned getGITPtrHigh() const
bool hasSpilledSGPRs() const
void addToPrologEpilogSGPRSpills(Register Reg, PrologEpilogSGPRSaveRestoreInfo SI)
Register getScratchSGPRCopyDstReg(Register Reg) const
void setScratchRSrcReg(Register Reg)
void reserveWWMRegister(Register Reg)
Register getFrameRegister(const MachineFunction &MF) const override
Represents a location in source code.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool test(unsigned Idx) const
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
Represent a constant reference to a string, i.e.
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
virtual bool hasReservedCallFrame(const MachineFunction &MF) const
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
void restoreCalleeSavedRegister(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const CalleeSavedInfo &CS, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an SmallVector or SmallString.
StringRef str() const
Return a StringRef for the vector contents.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ PRIVATE_ADDRESS
Address space for private memory.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI, unsigned DynamicVGPRBlockSize, std::optional< bool > EnableWavefrontSize32)
uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset)
Convert ByteOffset to dwords if the subtarget uses dword SMRD immediate offsets.
LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isCompute(CallingConv::ID CC)
LLVM_READNONE constexpr bool isChainCC(CallingConv::ID CC)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ ScalablePredicateVector
This is an optimization pass for GlobalISel generic memory operations.
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
constexpr RegState getKillRegState(bool B)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
void clearDebugInfoForSpillFIs(MachineFrameInfo &MFI, MachineBasicBlock &MBB, const BitVector &SpillFIs)
Replace frame index operands with null registers in debug value instructions for the specified spill ...
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
auto make_first_range(ContainerTy &&c)
Given a container of pairs, return a range over the first elements.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool is_sorted(R &&Range, Compare C)
Wrapper function around std::is_sorted to check if elements in a range R are sorted with respect to a...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
@ And
Bitwise or logical AND of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
MCRegisterClass TargetRegisterClass
@ PRIVATE_SEGMENT_WAVE_BYTE_OFFSET
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.