LLVM 24.0.0git
SIFrameLowering.cpp
Go to the documentation of this file.
1//===----------------------- SIFrameLowering.cpp --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8
9#include "SIFrameLowering.h"
10#include "AMDGPULaneMaskUtils.h"
11#include "GCNSubtarget.h"
14#include "SISpillUtils.h"
20#include "llvm/Support/LEB128.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "frame-info"
26
27static constexpr unsigned SGPRBitSize = 32;
28static constexpr unsigned SGPRByteSize = SGPRBitSize / 8;
29static constexpr unsigned VGPRLaneBitSize = 32;
30
31// The DWARF register numbers of vcc_lo and vcc_hi (VCC_32_LO and VCC_32_HI) are
32// only defined in wave32. In wave64 both halves are described as part of
33// VCC_64.
34static bool isWave64VCCHalf(const GCNSubtarget &ST, MCRegister Reg) {
35 return ST.isWave64() && (Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI);
36}
37
38// Find a register matching \p RC from \p LiveUnits which is unused and
39// available throughout the function. On failure, returns AMDGPU::NoRegister.
40// TODO: Rewrite the loop here to iterate over MCRegUnits instead of
41// MCRegisters. This should reduce the number of iterations and avoid redundant
42// checking.
44 const LiveRegUnits &LiveUnits,
45 const TargetRegisterClass &RC) {
46 for (MCRegister Reg : RC) {
47 if (!MRI.isPhysRegUsed(Reg) && LiveUnits.available(Reg) &&
48 !MRI.isReserved(Reg))
49 return Reg;
50 }
51 return MCRegister();
52}
53
54static void encodeDwarfRegisterLocation(int DwarfReg, raw_ostream &OS) {
55 assert(DwarfReg >= 0);
56 if (DwarfReg < 32) {
57 OS << uint8_t(dwarf::DW_OP_reg0 + DwarfReg);
58 } else {
59 OS << uint8_t(dwarf::DW_OP_regx);
60 encodeULEB128(DwarfReg, OS);
61 }
62}
63
65 int64_t DwarfStackPtrReg) {
66 assert(ST.hasFlatScratchEnabled());
67
68 // When flat scratch is enabled, the stack pointer is an address in the
69 // private_lane DWARF address space (i.e. swizzled), but in order to
70 // accurately and efficiently describe things like masked spills of vector
71 // registers we want to define the CFA to be an address in the private_wave
72 // DWARF address space (i.e. unswizzled). To achieve this we scale the stack
73 // pointer by the wavefront size, implemented as (SP << wave_size_log2).
74 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
75 assert(WavefrontSizeLog2 < 32);
76
79 encodeDwarfRegisterLocation(DwarfStackPtrReg, OSBlock);
80 OSBlock << uint8_t(dwarf::DW_OP_deref_size) << uint8_t(SGPRByteSize)
81 << uint8_t(dwarf::DW_OP_lit0 + WavefrontSizeLog2)
82 << uint8_t(dwarf::DW_OP_shl)
83 << uint8_t(dwarf::DW_OP_lit0 +
84 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave)
85 << uint8_t(dwarf::DW_OP_LLVM_user)
86 << uint8_t(dwarf::DW_OP_LLVM_form_aspace_address);
87
88 SmallString<20> CFIInst;
89 raw_svector_ostream OSCFIInst(CFIInst);
90 OSCFIInst << uint8_t(dwarf::DW_CFA_def_cfa_expression);
91 encodeULEB128(Block.size(), OSCFIInst);
92 OSCFIInst << Block;
93
94 return MCCFIInstruction::createEscape(nullptr, OSCFIInst.str());
95}
96
97void SIFrameLowering::emitDefCFA(MachineBasicBlock &MBB,
99 DebugLoc const &DL, MCRegister StackPtrReg,
100 bool AspaceAlreadyDefined,
101 MachineInstr::MIFlag Flags) const {
103 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
104 const SIRegisterInfo *TRI = ST.getRegisterInfo();
105
106 int64_t DwarfStackPtrReg = TRI->getDwarfRegNum(StackPtrReg, false);
107 MCCFIInstruction CFIInst =
108 ST.hasFlatScratchEnabled()
109 ? createScaledCFAInPrivateWave(ST, DwarfStackPtrReg)
110 : (AspaceAlreadyDefined
111 ? MCCFIInstruction::createLLVMDefAspaceCfa(
112 nullptr, DwarfStackPtrReg, 0,
113 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave, SMLoc())
114 : MCCFIInstruction::createDefCfaRegister(nullptr,
115 DwarfStackPtrReg));
116 buildCFI(MBB, MBBI, DL, CFIInst, Flags);
117}
118
119// Find a scratch register that we can use in the prologue. We avoid using
120// callee-save registers since they may appear to be free when this is called
121// from canUseAsPrologue (during shrink wrapping), but then no longer be free
122// when this is called from emitPrologue.
125 const TargetRegisterClass &RC, bool Unused = false) {
126 // Mark callee saved registers as used so we will not choose them.
127 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
128 for (unsigned i = 0; CSRegs[i]; ++i)
129 LiveUnits.addReg(CSRegs[i]);
130
131 // We are looking for a register that can be used throughout the entire
132 // function, so any use is unacceptable.
133 if (Unused)
134 return findUnusedRegister(MRI, LiveUnits, RC);
135
136 for (MCRegister Reg : RC) {
137 if (LiveUnits.available(Reg) && !MRI.isReserved(Reg))
138 return Reg;
139 }
140
141 return MCRegister();
142}
143
144/// Query target location for spilling SGPRs
145/// \p IncludeScratchCopy : Also look for free scratch SGPRs
147 MachineFunction &MF, LiveRegUnits &LiveUnits, Register SGPR,
148 const TargetRegisterClass &RC = AMDGPU::SReg_32_XM0_XEXECRegClass,
149 bool IncludeScratchCopy = true) {
151 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
152
153 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
154 const SIRegisterInfo *TRI = ST.getRegisterInfo();
155 unsigned Size = TRI->getSpillSize(RC);
156 Align Alignment = TRI->getSpillAlign(RC);
157
158 // We need to save and restore the given SGPR.
159
160 Register ScratchSGPR;
161 // 1: Try to save the given register into an unused scratch SGPR. The
162 // LiveUnits should have all the callee saved registers marked as used. For
163 // certain cases we skip copy to scratch SGPR.
164 if (IncludeScratchCopy)
165 ScratchSGPR = findUnusedRegister(MF.getRegInfo(), LiveUnits, RC);
166
167 if (!ScratchSGPR) {
168 int FI = FrameInfo.CreateStackObject(Size, Alignment, true, nullptr,
170
171 if (TRI->spillSGPRToVGPR() &&
172 MFI->allocateSGPRSpillToVGPRLane(MF, FI, /*SpillToPhysVGPRLane=*/true,
173 /*IsPrologEpilog=*/true)) {
174 // 2: There's no free lane to spill, and no free register to save the
175 // SGPR, so we're forced to take another VGPR to use for the spill.
179
180 LLVM_DEBUG(auto Spill = MFI->getSGPRSpillToPhysicalVGPRLanes(FI).front();
181 dbgs() << printReg(SGPR, TRI) << " requires fallback spill to "
182 << printReg(Spill.VGPR, TRI) << ':' << Spill.Lane
183 << '\n';);
184 } else {
185 // Remove dead <FI> index
187 // 3: If all else fails, spill the register to memory.
188 FI = FrameInfo.CreateSpillStackObject(Size, Alignment);
190 SGPR,
192 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling "
193 << printReg(SGPR, TRI) << '\n');
194 }
195 } else {
199 LiveUnits.addReg(ScratchSGPR);
200 LLVM_DEBUG(dbgs() << "Saving " << printReg(SGPR, TRI) << " with copy to "
201 << printReg(ScratchSGPR, TRI) << '\n');
202 }
203}
204
205// We need to specially emit stack operations here because a different frame
206// register is used than in the rest of the function, as getFrameRegister would
207// use.
208static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI,
209 const SIMachineFunctionInfo &FuncInfo,
210 LiveRegUnits &LiveUnits, MachineFunction &MF,
213 Register SpillReg, int FI, Register FrameReg,
214 int64_t DwordOff = 0) {
215 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
216 : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
217
218 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
221 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FI),
222 FrameInfo.getObjectAlign(FI));
223 LiveUnits.addReg(SpillReg);
224 bool IsKill = !MBB.isLiveIn(SpillReg);
225 TRI.buildSpillLoadStore(MBB, I, DL, Opc, FI, SpillReg, IsKill, FrameReg,
226 DwordOff, MMO, nullptr, &LiveUnits);
227 if (IsKill)
228 LiveUnits.removeReg(SpillReg);
229}
230
231static void buildEpilogRestore(const GCNSubtarget &ST,
232 const SIRegisterInfo &TRI,
233 const SIMachineFunctionInfo &FuncInfo,
234 LiveRegUnits &LiveUnits, MachineFunction &MF,
237 const DebugLoc &DL, Register SpillReg, int FI,
238 Register FrameReg, int64_t DwordOff = 0) {
239 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
240 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
241
242 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
245 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FI),
246 FrameInfo.getObjectAlign(FI));
247 TRI.buildSpillLoadStore(MBB, I, DL, Opc, FI, SpillReg, false, FrameReg,
248 DwordOff, MMO, nullptr, &LiveUnits);
249}
250
252 const DebugLoc &DL, const SIInstrInfo *TII,
253 Register TargetReg) {
254 MachineFunction *MF = MBB.getParent();
256 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
257 const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
258 Register TargetLo = TRI->getSubReg(TargetReg, AMDGPU::sub0);
259 Register TargetHi = TRI->getSubReg(TargetReg, AMDGPU::sub1);
260
261 if (MFI->getGITPtrHigh() != 0xffffffff) {
262 BuildMI(MBB, I, DL, SMovB32, TargetHi)
263 .addImm(MFI->getGITPtrHigh())
264 .addReg(TargetReg, RegState::ImplicitDefine);
265 } else {
266 const MCInstrDesc &GetPC64 = TII->get(AMDGPU::S_GETPC_B64_pseudo);
267 BuildMI(MBB, I, DL, GetPC64, TargetReg);
268 }
269 Register GitPtrLo = MFI->getGITPtrLoReg(*MF);
270 MF->getRegInfo().addLiveIn(GitPtrLo);
271 MBB.addLiveIn(GitPtrLo);
272 BuildMI(MBB, I, DL, SMovB32, TargetLo)
273 .addReg(GitPtrLo);
274}
275
276static void initLiveUnits(LiveRegUnits &LiveUnits, const SIRegisterInfo &TRI,
277 const SIMachineFunctionInfo *FuncInfo,
279 MachineBasicBlock::iterator MBBI, bool IsProlog) {
280 if (LiveUnits.empty()) {
281 LiveUnits.init(TRI);
282 if (IsProlog) {
283 LiveUnits.addLiveIns(MBB);
284 } else {
285 // In epilog.
286 LiveUnits.addLiveOuts(MBB);
287 LiveUnits.stepBackward(*MBBI);
288 }
289 }
290}
291
292namespace llvm {
293
294// SpillBuilder to save/restore special SGPR spills like the one needed for FP,
295// BP, etc. These spills are delayed until the current function's frame is
296// finalized. For a given register, the builder uses the
297// PrologEpilogSGPRSaveRestoreInfo to decide the spill method.
301 MachineFunction &MF;
302 const GCNSubtarget &ST;
303 MachineFrameInfo &MFI;
304 SIMachineFunctionInfo *FuncInfo;
305 const SIInstrInfo *TII;
306 const SIRegisterInfo &TRI;
307 const MCRegisterInfo *MCRI;
308 const SIFrameLowering *TFI;
309 Register SuperReg;
311 LiveRegUnits &LiveUnits;
312 const DebugLoc &DL;
313 Register FrameReg;
314 ArrayRef<int16_t> SplitParts;
315 unsigned NumSubRegs;
316 unsigned EltSize = 4;
317 bool IsFramePtrPrologSpill;
318 bool NeedsFrameMoves;
319
320 static bool isExec(Register Reg) {
321 return Reg == AMDGPU::EXEC_LO || Reg == AMDGPU::EXEC;
322 }
323
324 /// If this builder requires SuperReg-based CFI, which is emitted after all
325 /// SubRegs are actually spilled, return the Register which should be used
326 /// as input to getDwarfRegNum. Otherwise, CFI should be generated per-SubReg.
327 ///
328 /// Note: Most spills handled by this builder generate CFI after each
329 /// SubReg spill, as each SubReg maps directly to a CFI register via
330 /// getDwarfRegNum(SubReg, false). All other cases currently currently
331 /// correspond to the SuperReg directly.
332 MCRegister getCFISuperReg() const {
333 if (IsFramePtrPrologSpill)
334 return FuncInfo->getFrameOffsetReg();
335 // FIXME: CFI for EXEC needs a fix by accurately computing the spill
336 // offset for both the low and high components.
337 if (isExec(SuperReg))
338 return AMDGPU::EXEC;
339 return {};
340 }
341
342 void saveToMemory(const int FI) const {
343 MachineRegisterInfo &MRI = MF.getRegInfo();
344 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
345 assert(!MFI.isDeadObjectIndex(FI));
346
347 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MI, /*IsProlog*/ true);
348
350 MRI, LiveUnits, AMDGPU::VGPR_32RegClass);
351 if (!TmpVGPR)
352 report_fatal_error("failed to find free scratch register");
353
354 auto BuildCFI = [&](Register Reg) {
355 TFI->buildCFI(MBB, MI, DL,
357 nullptr, MCRI->getDwarfRegNum(Reg, false),
358 MFI.getObjectOffset(FI) * ST.getWavefrontSize()));
359 };
360 MCRegister CFISuperReg = getCFISuperReg();
361 for (unsigned I = 0, DwordOff = 0; I < NumSubRegs; ++I) {
362 Register SubReg = NumSubRegs == 1
363 ? SuperReg
364 : Register(TRI.getSubReg(SuperReg, SplitParts[I]));
365 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
366 .addReg(SubReg);
367
368 buildPrologSpill(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MI, DL, TmpVGPR,
369 FI, FrameReg, DwordOff);
370 if (NeedsFrameMoves && !CFISuperReg)
371 BuildCFI(SubReg);
372 DwordOff += 4;
373 }
374 if (NeedsFrameMoves && CFISuperReg)
375 BuildCFI(CFISuperReg);
376 }
377
378 void saveToVGPRLane(const int FI) const {
379 assert(!MFI.isDeadObjectIndex(FI));
380
381 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
383 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FI);
384 assert(Spill.size() == NumSubRegs);
385
386 MCRegister CFISuperReg = getCFISuperReg();
387 for (unsigned I = 0; I < NumSubRegs; ++I) {
388 Register SubReg = NumSubRegs == 1
389 ? SuperReg
390 : Register(TRI.getSubReg(SuperReg, SplitParts[I]));
391 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_S32_TO_VGPR),
392 Spill[I].VGPR)
393 .addReg(SubReg)
394 .addImm(Spill[I].Lane)
395 .addReg(Spill[I].VGPR, RegState::Undef);
396 if (NeedsFrameMoves && !CFISuperReg)
397 TFI->buildCFIForSGPRToVGPRSpill(MBB, MI, DL, SubReg, Spill[I].VGPR,
398 Spill[I].Lane);
399 }
400 if (NeedsFrameMoves && CFISuperReg)
401 TFI->buildCFIForSGPRToVGPRSpill(MBB, MI, DL, CFISuperReg, Spill);
402 }
403
404 void copyToScratchSGPR(Register DstReg) const {
405 BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), DstReg)
406 .addReg(SuperReg)
408 if (NeedsFrameMoves) {
409 MCRegister CFISuperReg = getCFISuperReg();
410 if (!CFISuperReg)
411 CFISuperReg = SuperReg;
412 if (isWave64VCCHalf(ST, DstReg)) {
413 assert(NumSubRegs == 1 && "Expected a 32-bit SGPR");
414 TFI->buildCFIForSGPRToVCCHalfCopy(MBB, MI, DL, CFISuperReg, DstReg);
415 } else {
416 TFI->buildCFIForSRegToSRegSpill(MBB, MI, DL, CFISuperReg, DstReg);
417 }
418 }
419 }
420
421 void restoreFromMemory(const int FI) {
422 MachineRegisterInfo &MRI = MF.getRegInfo();
423 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
424
425 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MI, /*IsProlog*/ false);
427 MRI, LiveUnits, AMDGPU::VGPR_32RegClass);
428 if (!TmpVGPR)
429 report_fatal_error("failed to find free scratch register");
430
431 for (unsigned I = 0, DwordOff = 0; I < NumSubRegs; ++I) {
432 MCRegister SubReg = NumSubRegs == 1
433 ? SuperReg.asMCReg()
434 : TRI.getSubReg(SuperReg, SplitParts[I]);
435
436 buildEpilogRestore(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MI, DL,
437 TmpVGPR, FI, FrameReg, DwordOff);
438 assert(SubReg.isPhysical());
439
440 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg)
441 .addReg(TmpVGPR, RegState::Kill);
442 DwordOff += 4;
443 }
444 }
445
446 void restoreFromVGPRLane(const int FI) {
447 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
449 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FI);
450 assert(Spill.size() == NumSubRegs);
451
452 for (unsigned I = 0; I < NumSubRegs; ++I) {
453 MCRegister SubReg = NumSubRegs == 1
454 ? SuperReg.asMCReg()
455 : TRI.getSubReg(SuperReg, SplitParts[I]);
456 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_RESTORE_S32_FROM_VGPR), SubReg)
457 .addReg(Spill[I].VGPR)
458 .addImm(Spill[I].Lane);
459 }
460 }
461
462 void copyFromScratchSGPR(Register SrcReg) const {
463 BuildMI(MBB, MI, DL, TII->get(AMDGPU::COPY), SuperReg)
464 .addReg(SrcReg)
466 }
467
468public:
473 const DebugLoc &DL, const SIInstrInfo *TII,
474 const SIRegisterInfo &TRI,
475 LiveRegUnits &LiveUnits, Register FrameReg,
476 bool IsFramePtrPrologSpill = false)
477 : MI(MI), MBB(MBB), MF(*MBB.getParent()),
478 ST(MF.getSubtarget<GCNSubtarget>()), MFI(MF.getFrameInfo()),
479 FuncInfo(MF.getInfo<SIMachineFunctionInfo>()), TII(TII), TRI(TRI),
480 MCRI(MF.getContext().getRegisterInfo()), TFI(ST.getFrameLowering()),
481 SuperReg(Reg), SI(SI), LiveUnits(LiveUnits), DL(DL), FrameReg(FrameReg),
482 IsFramePtrPrologSpill(IsFramePtrPrologSpill),
483 NeedsFrameMoves(MF.needsFrameMoves()) {
484 const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(SuperReg);
485 SplitParts = TRI.getRegSplitParts(RC, EltSize);
486 NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
487
488 assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
489 }
490
491 void save() {
492 switch (SI.getKind()) {
494 return saveToMemory(SI.getIndex());
496 return saveToVGPRLane(SI.getIndex());
498 return copyToScratchSGPR(SI.getReg());
499 }
500 }
501
502 void restore() {
503 switch (SI.getKind()) {
505 return restoreFromMemory(SI.getIndex());
507 return restoreFromVGPRLane(SI.getIndex());
509 return copyFromScratchSGPR(SI.getReg());
510 }
511 }
512};
513
514} // namespace llvm
515
516// Emit flat scratch setup code, assuming `MFI->hasFlatScratchInit()`
517void SIFrameLowering::emitEntryFunctionFlatScratchInit(
519 const DebugLoc &DL, Register ScratchWaveOffsetReg) const {
520 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
521 const SIInstrInfo *TII = ST.getInstrInfo();
522 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
523 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
524
525 // We don't need this if we only have spills since there is no user facing
526 // scratch.
527
528 // TODO: If we know we don't have flat instructions earlier, we can omit
529 // this from the input registers.
530 //
531 // TODO: We only need to know if we access scratch space through a flat
532 // pointer. Because we only detect if flat instructions are used at all,
533 // this will be used more often than necessary on VI.
534
535 Register FlatScrInitLo;
536 Register FlatScrInitHi;
537
538 if (ST.isAmdPalOS()) {
539 // Extract the scratch offset from the descriptor in the GIT
540 LiveRegUnits LiveUnits;
541 LiveUnits.init(*TRI);
542 LiveUnits.addLiveIns(MBB);
543
544 // Find unused reg to load flat scratch init into
545 MachineRegisterInfo &MRI = MF.getRegInfo();
546 Register FlatScrInit = AMDGPU::NoRegister;
547 ArrayRef<MCPhysReg> AllSGPR64s = TRI->getAllSGPR64(MF);
548 unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 1) / 2;
549 AllSGPR64s = AllSGPR64s.slice(
550 std::min(static_cast<unsigned>(AllSGPR64s.size()), NumPreloaded));
551 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
552 for (MCPhysReg Reg : AllSGPR64s) {
553 if (LiveUnits.available(Reg) && !MRI.isReserved(Reg) &&
554 MRI.isAllocatable(Reg) && !TRI->isSubRegisterEq(Reg, GITPtrLoReg)) {
555 FlatScrInit = Reg;
556 break;
557 }
558 }
559 assert(FlatScrInit && "Failed to find free register for scratch init");
560
561 FlatScrInitLo = TRI->getSubReg(FlatScrInit, AMDGPU::sub0);
562 FlatScrInitHi = TRI->getSubReg(FlatScrInit, AMDGPU::sub1);
563
564 buildGitPtr(MBB, I, DL, TII, FlatScrInit);
565
566 // We now have the GIT ptr - now get the scratch descriptor from the entry
567 // at offset 0 (or offset 16 for a compute shader).
568 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
569 const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
570 auto *MMO = MF.getMachineMemOperand(
571 PtrInfo,
574 8, Align(4));
575 unsigned Offset =
577 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
578 unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(Subtarget, Offset);
579 BuildMI(MBB, I, DL, LoadDwordX2, FlatScrInit)
580 .addReg(FlatScrInit)
581 .addImm(EncodedOffset) // offset
582 .addImm(0) // cpol
583 .addMemOperand(MMO);
584
585 // Mask the offset in [47:0] of the descriptor
586 const MCInstrDesc &SAndB32 = TII->get(AMDGPU::S_AND_B32);
587 auto And = BuildMI(MBB, I, DL, SAndB32, FlatScrInitHi)
588 .addReg(FlatScrInitHi)
589 .addImm(0xffff);
590 And->getOperand(3).setIsDead(); // Mark SCC as dead.
591 } else {
592 Register FlatScratchInitReg =
594 assert(FlatScratchInitReg);
595
596 MachineRegisterInfo &MRI = MF.getRegInfo();
597 MRI.addLiveIn(FlatScratchInitReg);
598 MBB.addLiveIn(FlatScratchInitReg);
599
600 FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0);
601 FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1);
602 }
603
604 // Do a 64-bit pointer add.
605 if (ST.flatScratchIsPointer()) {
606 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
607 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo)
608 .addReg(FlatScrInitLo)
609 .addReg(ScratchWaveOffsetReg);
610 auto Addc = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32),
611 FlatScrInitHi)
612 .addReg(FlatScrInitHi)
613 .addImm(0);
614 Addc->getOperand(3).setIsDead(); // Mark SCC as dead.
615
616 using namespace AMDGPU::Hwreg;
617 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SETREG_B32))
618 .addReg(FlatScrInitLo)
619 .addImm(int16_t(HwregEncoding::encode(ID_FLAT_SCR_LO, 0, 32)));
620 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SETREG_B32))
621 .addReg(FlatScrInitHi)
622 .addImm(int16_t(HwregEncoding::encode(ID_FLAT_SCR_HI, 0, 32)));
623 return;
624 }
625
626 // For GFX9.
627 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), AMDGPU::FLAT_SCR_LO)
628 .addReg(FlatScrInitLo)
629 .addReg(ScratchWaveOffsetReg);
630 auto Addc = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32),
631 AMDGPU::FLAT_SCR_HI)
632 .addReg(FlatScrInitHi)
633 .addImm(0);
634 Addc->getOperand(3).setIsDead(); // Mark SCC as dead.
635
636 return;
637 }
638
639 assert(ST.getGeneration() < AMDGPUSubtarget::GFX9);
640
641 // Copy the size in bytes.
642 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::FLAT_SCR_LO)
643 .addReg(FlatScrInitHi, RegState::Kill);
644
645 // Add wave offset in bytes to private base offset.
646 // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init.
647 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), FlatScrInitLo)
648 .addReg(FlatScrInitLo)
649 .addReg(ScratchWaveOffsetReg);
650
651 // Convert offset to 256-byte units.
652 auto LShr = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32),
653 AMDGPU::FLAT_SCR_HI)
654 .addReg(FlatScrInitLo, RegState::Kill)
655 .addImm(8);
656 LShr->getOperand(3).setIsDead(); // Mark SCC as dead.
657}
658
659// Note SGPRSpill stack IDs should only be used for SGPR spilling to VGPRs, not
660// memory. They should have been removed by now.
662 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
663 I != E; ++I) {
664 if (!MFI.isDeadObjectIndex(I))
665 return false;
666 }
667
668 return true;
669}
670
671// Shift down registers reserved for the scratch RSRC.
672Register SIFrameLowering::getEntryFunctionReservedScratchRsrcReg(
673 MachineFunction &MF) const {
674
675 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
676 const SIInstrInfo *TII = ST.getInstrInfo();
677 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
678 MachineRegisterInfo &MRI = MF.getRegInfo();
679 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
680
681 assert(MFI->isEntryFunction());
682
683 Register ScratchRsrcReg = MFI->getScratchRSrcReg();
684
685 if (!ScratchRsrcReg || (!MRI.isPhysRegUsed(ScratchRsrcReg) &&
687 return Register();
688
689 if (ST.hasSGPRInitBug() ||
690 ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
691 return ScratchRsrcReg;
692
693 // We reserved the last registers for this. Shift it down to the end of those
694 // which were actually used.
695 //
696 // FIXME: It might be safer to use a pseudoregister before replacement.
697
698 // FIXME: We should be able to eliminate unused input registers. We only
699 // cannot do this for the resources required for scratch access. For now we
700 // skip over user SGPRs and may leave unused holes.
701
702 unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4;
703 ArrayRef<MCPhysReg> AllSGPR128s = TRI->getAllSGPR128(MF);
704 AllSGPR128s = AllSGPR128s.slice(std::min(static_cast<unsigned>(AllSGPR128s.size()), NumPreloaded));
705
706 // Skip the last N reserved elements because they should have already been
707 // reserved for VCC etc.
708 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
709 for (MCPhysReg Reg : AllSGPR128s) {
710 // Pick the first unallocated one. Make sure we don't clobber the other
711 // reserved input we needed. Also for PAL, make sure we don't clobber
712 // the GIT pointer passed in SGPR0 or SGPR8.
714 (!GITPtrLoReg || !TRI->isSubRegisterEq(Reg, GITPtrLoReg))) {
715 MRI.replaceRegWith(ScratchRsrcReg, Reg);
718 return Reg;
719 }
720 }
721
722 return ScratchRsrcReg;
723}
724
725static unsigned getScratchScaleFactor(const GCNSubtarget &ST) {
726 return ST.hasFlatScratchEnabled() ? 1 : ST.getWavefrontSize();
727}
728
730 MachineBasicBlock &MBB) const {
731 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
732
733 // FIXME: If we only have SGPR spills, we won't actually be using scratch
734 // memory since these spill to VGPRs. We should be cleaning up these unused
735 // SGPR spill frame indices somewhere.
736
737 // FIXME: We still have implicit uses on SGPR spill instructions in case they
738 // need to spill to vector memory. It's likely that will not happen, but at
739 // this point it appears we need the setup. This part of the prolog should be
740 // emitted after frame indices are eliminated.
741
742 // FIXME: Remove all of the isPhysRegUsed checks
743
745 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
746 const SIInstrInfo *TII = ST.getInstrInfo();
747 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
749 const Function &F = MF.getFunction();
750 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
751
752 assert(MFI->isEntryFunction());
753
754 // Debug location must be unknown since the first debug location is used to
755 // determine the end of the prologue.
756 DebugLoc DL;
758
759 if (MF.needsFrameMoves()) {
760 // On entry the SP/FP are not set up, so we need to define the CFA in terms
761 // of a literal location expression.
762 static const char CFAEncodedInstUserOpsArr[] = {
763 dwarf::DW_CFA_def_cfa_expression,
764 4, // length
765 static_cast<char>(dwarf::DW_OP_lit0),
766 static_cast<char>(dwarf::DW_OP_lit0 +
767 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave),
768 static_cast<char>(dwarf::DW_OP_LLVM_user),
769 static_cast<char>(dwarf::DW_OP_LLVM_form_aspace_address)};
770 static StringRef CFAEncodedInstUserOps =
771 StringRef(CFAEncodedInstUserOpsArr, sizeof(CFAEncodedInstUserOpsArr));
772 buildCFI(MBB, I, DL,
773 MCCFIInstruction::createEscape(nullptr, CFAEncodedInstUserOps,
774 SMLoc(),
775 "CFA is 0 in private_wave aspace"));
776 // Unwinding halts when the return address (PC) is undefined.
777 buildCFI(MBB, I, DL,
779 nullptr, TRI->getDwarfRegNum(AMDGPU::PC_REG, false)));
780 }
781
782 Register PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
784
785 // We need to do the replacement of the private segment buffer register even
786 // if there are no stack objects. There could be stores to undef or a
787 // constant without an associated object.
788 //
789 // This will return `Register()` in cases where there are no actual
790 // uses of the SRSRC.
791 Register ScratchRsrcReg;
792 if (!ST.hasFlatScratchEnabled())
793 ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
794
795 // Make the selected register live throughout the function.
796 if (ScratchRsrcReg) {
797 for (MachineBasicBlock &OtherBB : MF) {
798 if (&OtherBB != &MBB) {
799 OtherBB.addLiveIn(ScratchRsrcReg);
800 }
801 }
802 }
803
804 // Now that we have fixed the reserved SRSRC we need to locate the
805 // (potentially) preloaded SRSRC.
806 Register PreloadedScratchRsrcReg;
807 if (ST.isAmdHsaOrMesa(F)) {
808 PreloadedScratchRsrcReg =
810 if (ScratchRsrcReg && PreloadedScratchRsrcReg) {
811 // We added live-ins during argument lowering, but since they were not
812 // used they were deleted. We're adding the uses now, so add them back.
813 MRI.addLiveIn(PreloadedScratchRsrcReg);
814 MBB.addLiveIn(PreloadedScratchRsrcReg);
815 }
816 }
817
818 // We found the SRSRC first because it needs four registers and has an
819 // alignment requirement. If the SRSRC that we found is clobbering with
820 // the scratch wave offset, which may be in a fixed SGPR or a free SGPR
821 // chosen by SITargetLowering::allocateSystemSGPRs, COPY the scratch
822 // wave offset to a free SGPR.
823 Register ScratchWaveOffsetReg;
824 if (PreloadedScratchWaveOffsetReg &&
825 TRI->isSubRegisterEq(ScratchRsrcReg, PreloadedScratchWaveOffsetReg)) {
826 ArrayRef<MCPhysReg> AllSGPRs = TRI->getAllSGPR32(MF);
827 unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
828 AllSGPRs = AllSGPRs.slice(
829 std::min(static_cast<unsigned>(AllSGPRs.size()), NumPreloaded));
830 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
831 for (MCPhysReg Reg : AllSGPRs) {
832 if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg) &&
833 !TRI->isSubRegisterEq(ScratchRsrcReg, Reg) && GITPtrLoReg != Reg) {
834 ScratchWaveOffsetReg = Reg;
835 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg)
836 .addReg(PreloadedScratchWaveOffsetReg, RegState::Kill);
837 break;
838 }
839 }
840
841 // FIXME: We can spill incoming arguments and restore at the end of the
842 // prolog.
843 if (!ScratchWaveOffsetReg)
845 "could not find temporary scratch offset register in prolog");
846 } else {
847 ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
848 }
849 assert(ScratchWaveOffsetReg || !PreloadedScratchWaveOffsetReg);
850
851 unsigned Offset = FrameInfo.getStackSize() * getScratchScaleFactor(ST);
852 if (!mayReserveScratchForCWSR(MF)) {
853 if (hasFP(MF)) {
855 assert(FPReg != AMDGPU::FP_REG);
856 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), FPReg).addImm(0);
857 }
858
861 assert(SPReg != AMDGPU::SP_REG);
862 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), SPReg).addImm(Offset);
863 }
864 } else {
865 // We need to check if we're on a compute queue - if we are, then the CWSR
866 // trap handler may need to store some VGPRs on the stack. The first VGPR
867 // block is saved separately, so we only need to allocate space for any
868 // additional VGPR blocks used. For now, we will make sure there's enough
869 // room for the theoretical maximum number of VGPRs that can be allocated.
870 // FIXME: Figure out if the shader uses fewer VGPRs in practice.
871 assert(hasFP(MF));
873 assert(FPReg != AMDGPU::FP_REG);
874 unsigned VGPRSize = llvm::alignTo(
875 (ST.getAddressableNumVGPRs(MFI->getDynamicVGPRBlockSize()) -
877 MFI->getDynamicVGPRBlockSize())) *
878 4,
879 FrameInfo.getMaxAlign());
881
882 BuildMI(MBB, I, DL, TII->get(AMDGPU::GET_STACK_BASE), FPReg);
885 assert(SPReg != AMDGPU::SP_REG);
886
887 // If at least one of the constants can be inlined, then we can use
888 // s_cselect. Otherwise, use a mov and cmovk.
889 if (AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm()) ||
891 ST.hasInv2PiInlineImm())) {
892 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CSELECT_B32), SPReg)
893 .addImm(Offset + VGPRSize)
894 .addImm(Offset);
895 } else {
896 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), SPReg).addImm(Offset);
897 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CMOVK_I32), SPReg)
898 .addImm(Offset + VGPRSize);
899 }
900 }
901 }
902
903 bool NeedsFlatScratchInit =
905 (MRI.isPhysRegUsed(AMDGPU::FLAT_SCR) || FrameInfo.hasCalls() ||
906 (!allStackObjectsAreDead(FrameInfo) && ST.hasFlatScratchEnabled()));
907
908 if ((NeedsFlatScratchInit || ScratchRsrcReg) &&
909 PreloadedScratchWaveOffsetReg && !ST.hasArchitectedFlatScratch()) {
910 MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
911 MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
912 }
913
914 if (NeedsFlatScratchInit) {
915 emitEntryFunctionFlatScratchInit(MF, MBB, I, DL, ScratchWaveOffsetReg);
916 }
917
918 if (ScratchRsrcReg) {
919 emitEntryFunctionScratchRsrcRegSetup(MF, MBB, I, DL,
920 PreloadedScratchRsrcReg,
921 ScratchRsrcReg, ScratchWaveOffsetReg);
922 }
923}
924
926 MachineBasicBlock &MBB) const {
927 // Entry functions don't usually need an epilogue. The only exception is a
928 // block ending in a chain call (which is the only possible tail call in an
929 // entry function), in which case the current frame is becoming dead and we
930 // need to reset SP to point to the base of the stack. Other exits of the
931 // function, e.g. blocks ending in S_ENDPGM, don't need any of this.
932 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
933 const SIInstrInfo *TII = ST.getInstrInfo();
934 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
935 assert(MBBI != MBB.end() && "return block should end in a terminator");
936 if (!TII->isTailCall(*MBBI))
937 return;
938
939 // If the prologue has set SP and there aren't any local variables, then we
940 // don't need to restore anything.
941 const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
942 if (requiresStackPointerReference(MF) && FrameInfo.getStackSize() == 0)
943 return;
944
946 const DebugLoc &DL = MBBI->getDebugLoc();
947
949 assert(SPReg != AMDGPU::SP_REG);
950
951 // We need to either reset SP to point before the local variables, or to
952 // initialize it in case we skipped it in the prologue. In either case,
953 // the SP needs to be either 0 or point right after the space reserved for
954 // CWSR (which the prologue conveniently puts in FP).
955 if (mayReserveScratchForCWSR(MF)) {
956 assert(hasFP(MF));
958 assert(FPReg != AMDGPU::FP_REG);
959 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), SPReg)
960 .addReg(FPReg)
962 } else {
963 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_MOV_B32), SPReg)
964 .addImm(0)
966 }
967}
968
969// Emit scratch RSRC setup code, assuming `ScratchRsrcReg != AMDGPU::NoReg`
970void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
972 const DebugLoc &DL, Register PreloadedScratchRsrcReg,
973 Register ScratchRsrcReg, Register ScratchWaveOffsetReg) const {
974
975 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
976 const SIInstrInfo *TII = ST.getInstrInfo();
977 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
979 const Function &Fn = MF.getFunction();
980
981 if (ST.isAmdPalOS()) {
982 // The pointer to the GIT is formed from the offset passed in and either
983 // the amdgpu-git-ptr-high function attribute or the top part of the PC
984 Register Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
985 Register Rsrc03 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
986
987 buildGitPtr(MBB, I, DL, TII, Rsrc01);
988
989 // We now have the GIT ptr - now get the scratch descriptor from the entry
990 // at offset 0 (or offset 16 for a compute shader).
992 const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
993 auto *MMO = MF.getMachineMemOperand(
994 PtrInfo,
997 16, Align(4));
998 unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
1000 unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(Subtarget, Offset);
1001 BuildMI(MBB, I, DL, LoadDwordX4, ScratchRsrcReg)
1002 .addReg(Rsrc01)
1003 .addImm(EncodedOffset) // offset
1004 .addImm(0) // cpol
1005 .addReg(ScratchRsrcReg, RegState::ImplicitDefine)
1006 .addMemOperand(MMO);
1007
1008 // The driver will always set the SRD for wave 64 (bits 118:117 of
1009 // descriptor / bits 22:21 of third sub-reg will be 0b11)
1010 // If the shader is actually wave32 we have to modify the const_index_stride
1011 // field of the descriptor 3rd sub-reg (bits 22:21) to 0b10 (stride=32). The
1012 // reason the driver does this is that there can be cases where it presents
1013 // 2 shaders with different wave size (e.g. VsFs).
1014 // TODO: convert to using SCRATCH instructions or multiple SRD buffers
1015 if (ST.isWave32()) {
1016 const MCInstrDesc &SBitsetB32 = TII->get(AMDGPU::S_BITSET0_B32);
1017 BuildMI(MBB, I, DL, SBitsetB32, Rsrc03)
1018 .addImm(21)
1019 .addReg(Rsrc03);
1020 }
1021 } else if (ST.isMesaGfxShader(Fn) || !PreloadedScratchRsrcReg) {
1022 assert(!ST.isAmdHsaOrMesa(Fn));
1023 const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
1024
1025 Register Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
1026 Register Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
1027
1028 // Use relocations to get the pointer, and setup the other bits manually.
1029 uint64_t Rsrc23 = TII->getScratchRsrcWords23();
1030
1032 Register Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
1033
1035 const MCInstrDesc &Mov64 = TII->get(AMDGPU::S_MOV_B64);
1036
1037 BuildMI(MBB, I, DL, Mov64, Rsrc01)
1039 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1040 } else {
1041 const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
1042
1043 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
1044 auto *MMO = MF.getMachineMemOperand(
1045 PtrInfo,
1048 8, Align(4));
1049 BuildMI(MBB, I, DL, LoadDwordX2, Rsrc01)
1051 .addImm(0) // offset
1052 .addImm(0) // cpol
1053 .addMemOperand(MMO)
1054 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1055 }
1056
1059 } else {
1060 Register Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
1061 Register Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
1062
1063 BuildMI(MBB, I, DL, SMovB32, Rsrc0)
1064 .addExternalSymbol("SCRATCH_RSRC_DWORD0")
1065 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1066
1067 BuildMI(MBB, I, DL, SMovB32, Rsrc1)
1068 .addExternalSymbol("SCRATCH_RSRC_DWORD1")
1069 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1070 }
1071
1072 BuildMI(MBB, I, DL, SMovB32, Rsrc2)
1073 .addImm(Lo_32(Rsrc23))
1074 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1075
1076 BuildMI(MBB, I, DL, SMovB32, Rsrc3)
1077 .addImm(Hi_32(Rsrc23))
1078 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1079 } else if (ST.isAmdHsaOrMesa(Fn)) {
1080 assert(PreloadedScratchRsrcReg);
1081
1082 if (ScratchRsrcReg != PreloadedScratchRsrcReg) {
1083 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
1084 .addReg(PreloadedScratchRsrcReg, RegState::Kill);
1085 }
1086 }
1087
1088 // Add the scratch wave offset into the scratch RSRC.
1089 //
1090 // We only want to update the first 48 bits, which is the base address
1091 // pointer, without touching the adjacent 16 bits of flags. We know this add
1092 // cannot carry-out from bit 47, otherwise the scratch allocation would be
1093 // impossible to fit in the 48-bit global address space.
1094 //
1095 // TODO: Evaluate if it is better to just construct an SRD using the flat
1096 // scratch init and some constants rather than update the one we are passed.
1097 Register ScratchRsrcSub0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
1098 Register ScratchRsrcSub1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
1099
1100 // We cannot Kill ScratchWaveOffsetReg here because we allow it to be used in
1101 // the kernel body via inreg arguments.
1102 BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), ScratchRsrcSub0)
1103 .addReg(ScratchRsrcSub0)
1104 .addReg(ScratchWaveOffsetReg)
1105 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1106 auto Addc = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), ScratchRsrcSub1)
1107 .addReg(ScratchRsrcSub1)
1108 .addImm(0)
1109 .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
1110 Addc->getOperand(3).setIsDead(); // Mark SCC as dead.
1111}
1112
1114 switch (ID) {
1118 return true;
1123 return false;
1124 }
1125 llvm_unreachable("Invalid TargetStackID::Value");
1126}
1127
1128void SIFrameLowering::emitPrologueEntryCFI(MachineBasicBlock &MBB,
1130 const DebugLoc &DL) const {
1131 const MachineFunction &MF = *MBB.getParent();
1132 const MachineRegisterInfo &MRI = MF.getRegInfo();
1133 const MCRegisterInfo *MCRI = MF.getContext().getRegisterInfo();
1134 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1135 const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo();
1136 MCRegister StackPtrReg =
1137 MF.getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg();
1138
1139 emitDefCFA(MBB, MBBI, DL, StackPtrReg, /*AspaceAlreadyDefined=*/true,
1141
1142 buildCFIForSRegToSRegSpill(MBB, MBBI, DL, AMDGPU::PC_REG,
1143 TRI.getReturnAddressReg(MF));
1144
1145 BitVector IsCalleeSaved(TRI.getNumRegs());
1146 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
1147 for (unsigned I = 0; CSRegs[I]; ++I) {
1148 IsCalleeSaved.set(CSRegs[I]);
1149 }
1150 auto ProcessReg = [&](MCPhysReg Reg) {
1151 // VCC is not preserved across calls.
1152 if (Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI)
1153 return;
1154 if (IsCalleeSaved.test(Reg) || !MRI.isPhysRegModified(Reg))
1155 return;
1156 unsigned DwarfReg = MCRI->getDwarfRegNum(Reg, false);
1157 buildCFI(MBB, MBBI, DL,
1158 MCCFIInstruction::createUndefined(nullptr, DwarfReg));
1159 };
1160
1161 // Emit CFI rules for caller saved Arch VGPRs which are clobbered
1162 unsigned NumArchVGPRs = ST.getAddressableNumArchVGPRs();
1163 for_each(AMDGPU::VGPR_32RegClass.getRegisters().take_front(NumArchVGPRs),
1164 ProcessReg);
1165
1166 // Emit CFI rules for caller saved Accum VGPRs which are clobbered
1167 if (ST.hasMAIInsts()) {
1168 for_each(AMDGPU::AGPR_32RegClass.getRegisters(), ProcessReg);
1169 }
1170
1171 // Emit CFI rules for caller saved SGPRs which are clobbered
1172 for_each(AMDGPU::SGPR_32RegClass.getRegisters(), ProcessReg);
1173}
1174
1175// Activate only the inactive lanes when \p EnableInactiveLanes is true.
1176// Otherwise, activate all lanes. It returns the saved exec.
1178 MachineFunction &MF,
1181 const DebugLoc &DL, bool IsProlog,
1182 bool EnableInactiveLanes) {
1183 Register ScratchExecCopy;
1185 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1186 const SIInstrInfo *TII = ST.getInstrInfo();
1187 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1189
1190 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, IsProlog);
1191
1192 if (FuncInfo->isWholeWaveFunction()) {
1193 // Whole wave functions already have a copy of the original EXEC mask that
1194 // we can use.
1195 assert(IsProlog && "Epilog should look at return, not setup");
1196 ScratchExecCopy =
1197 TII->getWholeWaveFunctionSetup(MF)->getOperand(0).getReg();
1198 assert(ScratchExecCopy && "Couldn't find copy of EXEC");
1199 } else {
1200 ScratchExecCopy = findScratchNonCalleeSaveRegister(
1201 MRI, LiveUnits, *TRI.getWaveMaskRegClass());
1202 }
1203
1204 if (!ScratchExecCopy)
1205 report_fatal_error("failed to find free scratch register");
1206
1207 LiveUnits.addReg(ScratchExecCopy);
1208
1209 const unsigned SaveExecOpc =
1210 ST.isWave32() ? (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B32
1211 : AMDGPU::S_OR_SAVEEXEC_B32)
1212 : (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B64
1213 : AMDGPU::S_OR_SAVEEXEC_B64);
1214 auto SaveExec =
1215 BuildMI(MBB, MBBI, DL, TII->get(SaveExecOpc), ScratchExecCopy).addImm(-1);
1216 SaveExec->getOperand(3).setIsDead(); // Mark SCC as dead.
1217
1218 return ScratchExecCopy;
1219}
1220
1224 LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy,
1225 const bool NeedsFrameMoves) const {
1227 MachineFrameInfo &MFI = MF.getFrameInfo();
1228 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1229 const SIInstrInfo *TII = ST.getInstrInfo();
1230 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1231 const MCRegisterInfo *MCRI = MF.getContext().getRegisterInfo();
1234
1235 // Spill Whole-Wave Mode VGPRs. Save only the inactive lanes of the scratch
1236 // registers. However, save all lanes of callee-saved VGPRs. Due to this, we
1237 // might end up flipping the EXEC bits twice.
1238 Register ScratchExecCopy;
1239 SmallVector<std::pair<Register, int>, 2> WWMCalleeSavedRegs, WWMScratchRegs;
1240 FuncInfo->splitWWMSpillRegisters(MF, WWMCalleeSavedRegs, WWMScratchRegs);
1241 if (!WWMScratchRegs.empty())
1242 ScratchExecCopy =
1243 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1244 /*IsProlog*/ true, /*EnableInactiveLanes*/ true);
1245
1246 auto StoreWWMRegisters =
1248 for (const auto &Reg : WWMRegs) {
1249 Register VGPR = Reg.first;
1250 int FI = Reg.second;
1251 buildPrologSpill(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MBBI, DL,
1252 VGPR, FI, FrameReg);
1253 if (NeedsFrameMoves) {
1254 // We spill the entire VGPR, so we can get away with just cfi_offset
1255 buildCFI(MBB, MBBI, DL,
1257 nullptr, MCRI->getDwarfRegNum(VGPR, false),
1258 MFI.getObjectOffset(FI) * ST.getWavefrontSize()));
1259 }
1260 }
1261 };
1262
1263 for (const Register Reg : make_first_range(WWMScratchRegs)) {
1264 if (!MRI.isReserved(Reg)) {
1265 MRI.addLiveIn(Reg);
1266 MBB.addLiveIn(Reg);
1267 }
1268 }
1269 StoreWWMRegisters(WWMScratchRegs);
1270
1271 auto EnableAllLanes = [&]() {
1272 BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), LMC.ExecReg).addImm(-1);
1273 };
1274
1275 if (!WWMCalleeSavedRegs.empty()) {
1276 if (ScratchExecCopy) {
1277 EnableAllLanes();
1278 } else {
1279 ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1280 /*IsProlog*/ true,
1281 /*EnableInactiveLanes*/ false);
1282 }
1283 }
1284
1285 StoreWWMRegisters(WWMCalleeSavedRegs);
1286 if (FuncInfo->isWholeWaveFunction()) {
1287 // If we have already saved some WWM CSR registers, then the EXEC is already
1288 // -1 and we don't need to do anything else. Otherwise, save the original
1289 // EXEC into the setup register and set EXEC to -1 here.
1290 if (!ScratchExecCopy)
1291 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL, /*IsProlog*/ true,
1292 /*EnableInactiveLanes*/ false);
1293 else if (WWMCalleeSavedRegs.empty())
1294 EnableAllLanes();
1295 } else if (ScratchExecCopy) {
1296 // FIXME: Split block and make terminator.
1297 BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), LMC.ExecReg)
1298 .addReg(ScratchExecCopy, RegState::Kill);
1299 LiveUnits.addReg(ScratchExecCopy);
1300 }
1301
1302 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1303
1304 for (const auto &Spill : FuncInfo->getPrologEpilogSGPRSpills()) {
1305 // Special handle FP spill:
1306 // Skip if FP is saved to a scratch SGPR, the save has already been emitted.
1307 // Otherwise, FP has been moved to a temporary register and spill it
1308 // instead.
1309 bool IsFramePtrPrologSpill = Spill.first == FramePtrReg;
1310 Register Reg = IsFramePtrPrologSpill ? FramePtrRegScratchCopy : Spill.first;
1311 if (!Reg)
1312 continue;
1313
1314 PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI,
1315 LiveUnits, FrameReg, IsFramePtrPrologSpill);
1316 SB.save();
1317 }
1318
1319 // If a copy to scratch SGPR has been chosen for any of the SGPR spills, make
1320 // such scratch registers live throughout the function.
1321 SmallVector<Register, 1> ScratchSGPRs;
1322 FuncInfo->getAllScratchSGPRCopyDstRegs(ScratchSGPRs);
1323 if (!ScratchSGPRs.empty()) {
1324 for (MachineBasicBlock &MBB : MF) {
1325 for (MCPhysReg Reg : ScratchSGPRs)
1326 MBB.addLiveIn(Reg);
1327
1328 MBB.sortUniqueLiveIns();
1329 }
1330 if (!LiveUnits.empty()) {
1331 for (MCPhysReg Reg : ScratchSGPRs)
1332 LiveUnits.addReg(Reg);
1333 }
1334 }
1335
1336 // Remove the spill entry created for EXEC. It is needed only for CFISaves in
1337 // the prologue.
1338 if (TRI.isCFISavedRegsSpillEnabled())
1339 FuncInfo->removePrologEpilogSGPRSpillEntry(TRI.getExec());
1340}
1341
1345 LiveRegUnits &LiveUnits, Register FrameReg,
1346 Register FramePtrRegScratchCopy) const {
1347 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1348 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1349 const SIInstrInfo *TII = ST.getInstrInfo();
1350 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1352 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1353
1354 for (const auto &Spill : FuncInfo->getPrologEpilogSGPRSpills()) {
1355 // Special handle FP restore:
1356 // Skip if FP needs to be restored from the scratch SGPR. Otherwise, restore
1357 // the FP value to a temporary register. The frame pointer should be
1358 // overwritten only at the end when all other spills are restored from
1359 // current frame.
1360 Register Reg =
1361 Spill.first == FramePtrReg ? FramePtrRegScratchCopy : Spill.first;
1362 if (!Reg)
1363 continue;
1364
1365 PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI,
1366 LiveUnits, FrameReg);
1367 SB.restore();
1368 }
1369
1370 // Restore Whole-Wave Mode VGPRs. Restore only the inactive lanes of the
1371 // scratch registers. However, restore all lanes of callee-saved VGPRs. Due to
1372 // this, we might end up flipping the EXEC bits twice.
1373 Register ScratchExecCopy;
1374 SmallVector<std::pair<Register, int>, 2> WWMCalleeSavedRegs, WWMScratchRegs;
1375 FuncInfo->splitWWMSpillRegisters(MF, WWMCalleeSavedRegs, WWMScratchRegs);
1376 auto RestoreWWMRegisters =
1378 for (const auto &Reg : WWMRegs) {
1379 Register VGPR = Reg.first;
1380 int FI = Reg.second;
1381 buildEpilogRestore(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MBBI, DL,
1382 VGPR, FI, FrameReg);
1383 }
1384 };
1385
1386 if (FuncInfo->isWholeWaveFunction()) {
1387 // For whole wave functions, the EXEC is already -1 at this point.
1388 // Therefore, we can restore the CSR WWM registers right away.
1389 RestoreWWMRegisters(WWMCalleeSavedRegs);
1390
1391 // The original EXEC is the first operand of the return instruction.
1392 MachineInstr &Return = MBB.instr_back();
1393 unsigned Opcode = Return.getOpcode();
1394 switch (Opcode) {
1395 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
1396 Opcode = AMDGPU::SI_RETURN;
1397 break;
1398 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
1399 Opcode = AMDGPU::SI_TCRETURN_GFX;
1400 break;
1401 default:
1402 llvm_unreachable("Unexpected return inst");
1403 }
1404 Register OrigExec = Return.getOperand(0).getReg();
1405
1406 if (!WWMScratchRegs.empty()) {
1407 BuildMI(MBB, MBBI, DL, TII->get(LMC.XorOpc), LMC.ExecReg)
1408 .addReg(OrigExec)
1409 .addImm(-1);
1410 RestoreWWMRegisters(WWMScratchRegs);
1411 }
1412
1413 // Restore original EXEC.
1414 BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), LMC.ExecReg).addReg(OrigExec);
1415
1416 // Drop the first operand and update the opcode.
1417 Return.removeOperand(0);
1418 Return.setDesc(TII->get(Opcode));
1419
1420 return;
1421 }
1422
1423 if (!WWMScratchRegs.empty()) {
1424 ScratchExecCopy =
1425 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1426 /*IsProlog=*/false, /*EnableInactiveLanes=*/true);
1427 }
1428 RestoreWWMRegisters(WWMScratchRegs);
1429 if (!WWMCalleeSavedRegs.empty()) {
1430 if (ScratchExecCopy) {
1431 BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), LMC.ExecReg).addImm(-1);
1432 } else {
1433 ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1434 /*IsProlog*/ false,
1435 /*EnableInactiveLanes*/ false);
1436 }
1437 }
1438
1439 RestoreWWMRegisters(WWMCalleeSavedRegs);
1440 if (ScratchExecCopy) {
1441 // FIXME: Split block and make terminator.
1442 BuildMI(MBB, MBBI, DL, TII->get(LMC.MovOpc), LMC.ExecReg)
1443 .addReg(ScratchExecCopy, RegState::Kill);
1444 }
1445}
1446
1448 MachineBasicBlock &MBB) const {
1450 if (FuncInfo->isEntryFunction()) {
1452 return;
1453 }
1454
1455 MachineFrameInfo &MFI = MF.getFrameInfo();
1456 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1457 const SIInstrInfo *TII = ST.getInstrInfo();
1458 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1460
1461 Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
1462 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1463 Register BasePtrReg =
1464 TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
1465 LiveRegUnits LiveUnits;
1466
1468 // DebugLoc must be unknown since the first instruction with DebugLoc is used
1469 // to determine the end of the prologue.
1470 DebugLoc DL;
1471
1472 bool HasFP = false;
1473 bool HasBP = false;
1474 int64_t NumBytes = MFI.getStackSize();
1475 int64_t RoundedSize = NumBytes;
1476
1477 // Functions that never return don't need to save and restore the FP or BP.
1478 const Function &F = MF.getFunction();
1479 bool SavesStackRegs =
1480 !F.hasFnAttribute(Attribute::NoReturn) && !FuncInfo->isChainFunction();
1481
1482 const bool NeedsFrameMoves = MF.needsFrameMoves();
1483
1484 if (NeedsFrameMoves)
1485 emitPrologueEntryCFI(MBB, MBBI, DL);
1486
1487 if (TRI.hasStackRealignment(MF))
1488 HasFP = true;
1489
1490 Register FramePtrRegScratchCopy;
1491 if (!HasFP && !hasFP(MF)) {
1492 // Emit the CSR spill stores with SP base register.
1493 emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, StackPtrReg,
1494 FramePtrRegScratchCopy, NeedsFrameMoves);
1495 } else if (SavesStackRegs) {
1496 // CSR spill stores will use FP as base register.
1497 Register SGPRForFPSaveRestoreCopy =
1498 FuncInfo->getScratchSGPRCopyDstReg(FramePtrReg);
1499
1500 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ true);
1501 if (SGPRForFPSaveRestoreCopy) {
1502 // Copy FP to the scratch register now and emit the CFI entry. It avoids
1503 // the extra FP copy needed in the other two cases when FP is spilled to
1504 // memory or to a VGPR lane.
1506 FramePtrReg,
1507 FuncInfo->getPrologEpilogSGPRSaveRestoreInfo(FramePtrReg), MBB, MBBI,
1508 DL, TII, TRI, LiveUnits, FramePtrReg,
1509 /*IsFramePtrPrologSpill*/ true);
1510 SB.save();
1511 LiveUnits.addReg(SGPRForFPSaveRestoreCopy);
1512 } else {
1513 // Copy FP into a new scratch register so that its previous value can be
1514 // spilled after setting up the new frame.
1515 FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister(
1516 MRI, LiveUnits, AMDGPU::SReg_32_XM0_XEXECRegClass);
1517 if (!FramePtrRegScratchCopy)
1518 report_fatal_error("failed to find free scratch register");
1519
1520 LiveUnits.addReg(FramePtrRegScratchCopy);
1521 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrRegScratchCopy)
1522 .addReg(FramePtrReg);
1523 }
1524 }
1525
1526 if (HasFP) {
1527 const unsigned Alignment = MFI.getMaxAlign().value();
1528
1529 RoundedSize += Alignment;
1530 if (LiveUnits.empty()) {
1531 LiveUnits.init(TRI);
1532 LiveUnits.addLiveIns(MBB);
1533 }
1534
1535 // s_add_i32 s33, s32, NumBytes
1536 // s_and_b32 s33, s33, 0b111...0000
1537 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_I32), FramePtrReg)
1538 .addReg(StackPtrReg)
1539 .addImm((Alignment - 1) * getScratchScaleFactor(ST))
1541 auto And = BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_AND_B32), FramePtrReg)
1542 .addReg(FramePtrReg, RegState::Kill)
1543 .addImm(-Alignment * getScratchScaleFactor(ST))
1545 And->getOperand(3).setIsDead(); // Mark SCC as dead.
1546 FuncInfo->setIsStackRealigned(true);
1547 } else if ((HasFP = hasFP(MF))) {
1548 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
1549 .addReg(StackPtrReg)
1551 }
1552
1553 // If FP is used, emit the CSR spills with FP base register.
1554 if (HasFP) {
1555 emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, FramePtrReg,
1556 FramePtrRegScratchCopy, NeedsFrameMoves);
1557 if (FramePtrRegScratchCopy)
1558 LiveUnits.removeReg(FramePtrRegScratchCopy);
1559 }
1560
1561 // If we need a base pointer, set it up here. It's whatever the value of
1562 // the stack pointer is at this point. Any variable size objects will be
1563 // allocated after this, so we can still use the base pointer to reference
1564 // the incoming arguments.
1565 if ((HasBP = TRI.hasBasePointer(MF))) {
1566 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), BasePtrReg)
1567 .addReg(StackPtrReg)
1569 }
1570
1571 if (HasFP) {
1572 if (NeedsFrameMoves)
1573 emitDefCFA(MBB, MBBI, DL, FramePtrReg, /*AspaceAlreadyDefined=*/false,
1575 }
1576
1577 if (HasFP && RoundedSize != 0) {
1578 auto Add = BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_I32), StackPtrReg)
1579 .addReg(StackPtrReg)
1580 .addImm(RoundedSize * getScratchScaleFactor(ST))
1582 Add->getOperand(3).setIsDead(); // Mark SCC as dead.
1583 }
1584
1585 bool FPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(FramePtrReg);
1586 (void)FPSaved;
1587 assert((!HasFP || FPSaved || !SavesStackRegs) &&
1588 "Needed to save FP but didn't save it anywhere");
1589
1590 // If we allow spilling to AGPRs we may have saved FP but then spill
1591 // everything into AGPRs instead of the stack.
1592 assert((HasFP || !FPSaved || !SavesStackRegs ||
1593 ST.getCLOpts().spill_vgpr_to_agpr) &&
1594 "Saved FP but didn't need it");
1595
1596 bool BPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(BasePtrReg);
1597 (void)BPSaved;
1598 assert((!HasBP || BPSaved || !SavesStackRegs) &&
1599 "Needed to save BP but didn't save it anywhere");
1600
1601 assert((HasBP || !BPSaved) && "Saved BP but didn't need it");
1602
1603 if (FuncInfo->isWholeWaveFunction()) {
1604 // SI_WHOLE_WAVE_FUNC_SETUP has outlived its purpose.
1605 TII->getWholeWaveFunctionSetup(MF)->eraseFromParent();
1606 }
1607}
1608
1610 MachineBasicBlock &MBB) const {
1611 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1612 if (FuncInfo->isEntryFunction()) {
1614 return;
1615 }
1616
1617 const MachineFrameInfo &MFI = MF.getFrameInfo();
1618 if (FuncInfo->isChainFunction() && !MFI.hasTailCall())
1619 return;
1620
1621 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1622 const SIInstrInfo *TII = ST.getInstrInfo();
1623 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1625 LiveRegUnits LiveUnits;
1626 // Get the insert location for the epilogue. If there were no terminators in
1627 // the block, get the last instruction.
1629 DebugLoc DL;
1630 if (!MBB.empty()) {
1631 MBBI = MBB.getLastNonDebugInstr();
1632 if (MBBI != MBB.end())
1633 DL = MBBI->getDebugLoc();
1634
1635 MBBI = MBB.getFirstTerminator();
1636 }
1637
1638 uint32_t NumBytes = MFI.getStackSize();
1639 uint32_t RoundedSize = FuncInfo->isStackRealigned()
1640 ? NumBytes + MFI.getMaxAlign().value()
1641 : NumBytes;
1642 const Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
1643 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1644 bool FPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(FramePtrReg);
1645
1646 if (RoundedSize != 0) {
1647 if (TRI.hasBasePointer(MF)) {
1648 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), StackPtrReg)
1649 .addReg(TRI.getBaseRegister())
1651 } else if (hasFP(MF)) {
1652 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), StackPtrReg)
1653 .addReg(FramePtrReg)
1655 }
1656 }
1657
1658 Register FramePtrRegScratchCopy;
1659 Register SGPRForFPSaveRestoreCopy =
1660 FuncInfo->getScratchSGPRCopyDstReg(FramePtrReg);
1661 if (FPSaved) {
1662 // CSR spill restores should use FP as base register. If
1663 // SGPRForFPSaveRestoreCopy is not true, restore the previous value of FP
1664 // into a new scratch register and copy to FP later when other registers are
1665 // restored from the current stack frame.
1666 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ false);
1667 if (SGPRForFPSaveRestoreCopy) {
1668 LiveUnits.addReg(SGPRForFPSaveRestoreCopy);
1669 } else {
1670 FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister(
1671 MRI, LiveUnits, AMDGPU::SReg_32_XM0_XEXECRegClass);
1672 if (!FramePtrRegScratchCopy)
1673 report_fatal_error("failed to find free scratch register");
1674
1675 LiveUnits.addReg(FramePtrRegScratchCopy);
1676 }
1677
1678 emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, FramePtrReg,
1679 FramePtrRegScratchCopy);
1680 }
1681
1682 if (hasFP(MF) && MF.needsFrameMoves()) {
1683 emitDefCFA(MBB, MBBI, DL, StackPtrReg, /*AspaceAlreadyDefined=*/false,
1685 }
1686
1687 if (FPSaved) {
1688 // Insert the copy to restore FP.
1689 Register SrcReg = SGPRForFPSaveRestoreCopy ? SGPRForFPSaveRestoreCopy
1690 : FramePtrRegScratchCopy;
1692 BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
1693 .addReg(SrcReg);
1694 if (SGPRForFPSaveRestoreCopy)
1696 } else {
1697 // Insert the CSR spill restores with SP as the base register.
1698 emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, StackPtrReg,
1699 FramePtrRegScratchCopy);
1700 }
1701}
1702
1703#ifndef NDEBUG
1705 const MachineFrameInfo &MFI = MF.getFrameInfo();
1706 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1707 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
1708 I != E; ++I) {
1709 if (!MFI.isDeadObjectIndex(I) &&
1712 return false;
1713 }
1714 }
1715
1716 return true;
1717}
1718#endif
1719
1721 int FI,
1722 Register &FrameReg) const {
1723 const SIRegisterInfo *RI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1724
1725 FrameReg = RI->getFrameRegister(MF);
1727}
1728
1730 MachineFunction &MF,
1731 RegScavenger *RS) const {
1732 MachineFrameInfo &MFI = MF.getFrameInfo();
1733
1734 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1735 const SIInstrInfo *TII = ST.getInstrInfo();
1736 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1739
1740 const bool SpillVGPRToAGPR = ST.hasMAIInsts() &&
1741 FuncInfo->hasSpilledVGPRs() &&
1742 ST.getCLOpts().spill_vgpr_to_agpr;
1743
1744 if (SpillVGPRToAGPR) {
1745 // To track the spill frame indices handled in this pass.
1746 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
1747 BitVector NonVGPRSpillFIs(MFI.getObjectIndexEnd(), false);
1748
1749 bool SeenDbgInstr = false;
1750
1751 for (MachineBasicBlock &MBB : MF) {
1753 int FrameIndex;
1754 if (MI.isDebugInstr())
1755 SeenDbgInstr = true;
1756
1757 if (TII->isVGPRSpill(MI)) {
1758 // Try to eliminate stack used by VGPR spills before frame
1759 // finalization.
1760 unsigned FIOp = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
1761 AMDGPU::OpName::vaddr);
1762 int FI = MI.getOperand(FIOp).getIndex();
1763 Register VReg =
1764 TII->getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
1765 if (FuncInfo->allocateVGPRSpillToAGPR(MF, FI,
1766 TRI->isAGPR(MRI, VReg))) {
1767 assert(RS != nullptr);
1768 RS->enterBasicBlockEnd(MBB);
1769 RS->backward(std::next(MI.getIterator()));
1770 TRI->eliminateFrameIndex(MI, 0, FIOp, RS);
1771 SpillFIs.set(FI);
1772 continue;
1773 }
1774 } else if (TII->isStoreToStackSlot(MI, FrameIndex) ||
1775 TII->isLoadFromStackSlot(MI, FrameIndex))
1776 if (!MFI.isFixedObjectIndex(FrameIndex))
1777 NonVGPRSpillFIs.set(FrameIndex);
1778 }
1779 }
1780
1781 // Stack slot coloring may assign different objects to the same stack slot.
1782 // If not, then the VGPR to AGPR spill slot is dead.
1783 for (unsigned FI : SpillFIs.set_bits())
1784 if (!NonVGPRSpillFIs.test(FI))
1785 FuncInfo->setVGPRToAGPRSpillDead(FI);
1786
1787 for (MachineBasicBlock &MBB : MF) {
1788 for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs())
1789 MBB.addLiveIn(Reg);
1790
1791 for (MCPhysReg Reg : FuncInfo->getAGPRSpillVGPRs())
1792 MBB.addLiveIn(Reg);
1793
1794 MBB.sortUniqueLiveIns();
1795
1796 if (!SpillFIs.empty() && SeenDbgInstr)
1797 clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
1798 }
1799 }
1800
1801 // At this point we've already allocated all spilled SGPRs to VGPRs if we
1802 // can. Any remaining SGPR spills will go to memory, so move them back to the
1803 // default stack.
1804 bool HaveSGPRToVMemSpill =
1805 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ true);
1807 "SGPR spill should have been removed in SILowerSGPRSpills");
1808
1809 // FIXME: The other checks should be redundant with allStackObjectsAreDead,
1810 // but currently hasNonSpillStackObjects is set only from source
1811 // allocas. Stack temps produced from legalization are not counted currently.
1812 if (!allStackObjectsAreDead(MFI)) {
1813 assert(RS && "RegScavenger required if spilling");
1814
1815 // Add an emergency spill slot
1816 RS->addScavengingFrameIndex(FuncInfo->getScavengeFI(MFI, *TRI));
1817
1818 if (HaveSGPRToVMemSpill && FuncInfo->hasNoWWMPoolSGPRSpillFallback()) {
1819 // The no-WWM-pool fallback can reach SGPR-to-memory lowering while an
1820 // ordinary frame-index scavenge is live. It may then need one slot for
1821 // its temporary VGPR and another for recursive address materialization.
1822 RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
1823 RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
1824 } else if (HaveSGPRToVMemSpill &&
1826 // Existing large-frame SGPR-to-memory spills need one additional VGPR
1827 // emergency frame index.
1828 RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
1829 }
1830 }
1831}
1832
1834 MachineFunction &MF, RegScavenger *RS) const {
1835 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1836 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1839
1840 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
1841 // On gfx908, we had initially reserved highest available VGPR for AGPR
1842 // copy. Now since we are done with RA, check if there exist an unused VGPR
1843 // which is lower than the eariler reserved VGPR before RA. If one exist,
1844 // use it for AGPR copy instead of one reserved before RA.
1845 Register VGPRForAGPRCopy = FuncInfo->getVGPRForAGPRCopy();
1846 Register UnusedLowVGPR =
1847 TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
1848 if (UnusedLowVGPR && (TRI->getHWRegIndex(UnusedLowVGPR) <
1849 TRI->getHWRegIndex(VGPRForAGPRCopy))) {
1850 // Reserve this newly identified VGPR (for AGPR copy)
1851 // reserved registers should already be frozen at this point
1852 // so we can avoid calling MRI.freezeReservedRegs and just use
1853 // MRI.reserveReg
1854 FuncInfo->setVGPRForAGPRCopy(UnusedLowVGPR);
1855 MRI.reserveReg(UnusedLowVGPR, TRI);
1856 }
1857 }
1858 // We initally reserved the highest available SGPR pair for long branches
1859 // now, after RA, we shift down to a lower unused one if one exists
1860 Register LongBranchReservedReg = FuncInfo->getLongBranchReservedReg();
1861 Register UnusedLowSGPR =
1862 TRI->findUnusedRegister(MRI, &AMDGPU::SGPR_64RegClass, MF);
1863 // If LongBranchReservedReg is null then we didn't find a long branch
1864 // and never reserved a register to begin with so there is nothing to
1865 // shift down. Then if UnusedLowSGPR is null, there isn't available lower
1866 // register to use so just keep the original one we set.
1867 if (LongBranchReservedReg && UnusedLowSGPR) {
1868 FuncInfo->setLongBranchReservedReg(UnusedLowSGPR);
1869 MRI.reserveReg(UnusedLowSGPR, TRI);
1870 }
1871}
1872
1873// The special SGPR spills like the one needed for FP, BP or any reserved
1874// registers delayed until frame lowering.
1876 MachineFunction &MF, BitVector &SavedVGPRs,
1877 bool NeedExecCopyReservedReg) const {
1878 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
1881 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1882 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1883 LiveRegUnits LiveUnits;
1884 LiveUnits.init(*TRI);
1885 // Initially mark callee saved registers as used so we will not choose them
1886 // while looking for scratch SGPRs.
1887 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
1888 for (unsigned I = 0; CSRegs[I]; ++I)
1889 LiveUnits.addReg(CSRegs[I]);
1890
1891 const TargetRegisterClass &RC = *TRI->getWaveMaskRegClass();
1892
1893 Register ReservedRegForExecCopy = MFI->getSGPRForEXECCopy();
1894 if (NeedExecCopyReservedReg ||
1895 (ReservedRegForExecCopy &&
1896 MRI.isPhysRegUsed(ReservedRegForExecCopy, /*SkipRegMaskTest=*/true))) {
1897 MRI.reserveReg(ReservedRegForExecCopy, TRI);
1898 Register UnusedScratchReg = findUnusedRegister(MRI, LiveUnits, RC);
1899 if (UnusedScratchReg) {
1900 // If found any unused scratch SGPR, reserve the register itself for Exec
1901 // copy and there is no need for any spills in that case.
1902 MFI->setSGPRForEXECCopy(UnusedScratchReg);
1903 MRI.replaceRegWith(ReservedRegForExecCopy, UnusedScratchReg);
1904 LiveUnits.addReg(UnusedScratchReg);
1905 } else {
1906 // Needs spill.
1907 assert(!MFI->hasPrologEpilogSGPRSpillEntry(ReservedRegForExecCopy) &&
1908 "Re-reserving spill slot for EXEC copy register");
1909 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, ReservedRegForExecCopy, RC,
1910 /*IncludeScratchCopy=*/false);
1911 }
1912 } else if (ReservedRegForExecCopy) {
1913 // Reset it at this point. There are no whole-wave copies and spills
1914 // encountered.
1915 MFI->setSGPRForEXECCopy(AMDGPU::NoRegister);
1916 }
1917
1918 if (TRI->isCFISavedRegsSpillEnabled()) {
1919 Register Exec = TRI->getExec();
1921 "Re-reserving spill slot for EXEC");
1922 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, Exec, RC);
1923 }
1924
1925 // Functions that don't return to the caller don't need to preserve
1926 // the FP and BP.
1927 const Function &F = MF.getFunction();
1928 if (F.hasFnAttribute(Attribute::NoReturn) ||
1929 AMDGPU::isChainCC(F.getCallingConv()))
1930 return;
1931
1932 // hasFP only knows about stack objects that already exist. We're now
1933 // determining the stack slots that will be created, so we have to predict
1934 // them. Stack objects force FP usage with calls.
1935 //
1936 // Note a new VGPR CSR may be introduced if one is used for the spill, but we
1937 // don't want to report it here.
1938 //
1939 // FIXME: Is this really hasReservedCallFrame?
1940 const bool WillHaveFP =
1941 FrameInfo.hasCalls() &&
1942 (SavedVGPRs.any() || !allStackObjectsAreDead(FrameInfo));
1943
1944 if (WillHaveFP || hasFP(MF)) {
1945 Register FramePtrReg = MFI->getFrameOffsetReg();
1946 assert(!MFI->hasPrologEpilogSGPRSpillEntry(FramePtrReg) &&
1947 "Re-reserving spill slot for FP");
1948 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, FramePtrReg);
1949 }
1950
1951 if (TRI->hasBasePointer(MF)) {
1952 Register BasePtrReg = TRI->getBaseRegister();
1953 assert(!MFI->hasPrologEpilogSGPRSpillEntry(BasePtrReg) &&
1954 "Re-reserving spill slot for BP");
1955 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, BasePtrReg);
1956 }
1957}
1958
1959// Only report VGPRs to generic code.
1961 BitVector &SavedVGPRs,
1962 RegScavenger *RS) const {
1964
1965 // If this is a function with the amdgpu_cs_chain[_preserve] calling
1966 // convention and it doesn't contain any calls to llvm.amdgcn.cs.chain, then
1967 // we don't need to save and restore anything.
1968 if (MFI->isChainFunction() && !MF.getFrameInfo().hasTailCall())
1969 return;
1970
1972
1973 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1974 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1975 const SIInstrInfo *TII = ST.getInstrInfo();
1976 bool NeedExecCopyReservedReg = false;
1977
1978 MachineInstr *ReturnMI = nullptr;
1979 for (MachineBasicBlock &MBB : MF) {
1980 for (MachineInstr &MI : MBB) {
1981 // TODO: Walking through all MBBs here would be a bad heuristic. Better
1982 // handle them elsewhere.
1983 if (TII->isWWMRegSpillOpcode(MI.getOpcode()))
1984 NeedExecCopyReservedReg = true;
1985 else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
1986 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1987 MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
1988 (MFI->isChainFunction() &&
1989 TII->isChainCallOpcode(MI.getOpcode()))) {
1990 // We expect all return to be the same size.
1991 assert(!ReturnMI ||
1992 (count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
1993 count_if(ReturnMI->operands(), [](auto Op) { return Op.isReg(); })));
1994 ReturnMI = &MI;
1995 }
1996 }
1997 }
1998
1999 SmallVector<Register> SortedWWMVGPRs;
2000 for (Register Reg : MFI->getWWMReservedRegs()) {
2001 // The shift-back is needed only for the VGPRs used for SGPR spills and they
2002 // are of 32-bit size. SIPreAllocateWWMRegs pass can add tuples into WWM
2003 // reserved registers.
2004 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
2005 if (TRI->getRegSizeInBits(*RC) != 32)
2006 continue;
2007 SortedWWMVGPRs.push_back(Reg);
2008 }
2009
2010 sort(SortedWWMVGPRs, std::greater<Register>());
2011 MFI->shiftWwmVGPRsToLowestRange(MF, SortedWWMVGPRs, SavedVGPRs);
2012
2013 if (MFI->isEntryFunction())
2014 return;
2015
2016 if (MFI->isWholeWaveFunction()) {
2017 // In practice, all the VGPRs are WWM registers, and we will need to save at
2018 // least their inactive lanes. Add them to WWMReservedRegs.
2019 assert(!NeedExecCopyReservedReg &&
2020 "Whole wave functions can use the reg mapped for their i1 argument");
2021
2022 unsigned NumArchVGPRs = ST.getAddressableNumArchVGPRs();
2023 for (MCRegister Reg :
2024 AMDGPU::VGPR_32RegClass.getRegisters().take_front(NumArchVGPRs))
2025 if (MF.getRegInfo().isPhysRegModified(Reg)) {
2026 MFI->reserveWWMRegister(Reg);
2027 MF.begin()->addLiveIn(Reg);
2028 }
2029 MF.begin()->sortUniqueLiveIns();
2030 }
2031
2032 // Remove any VGPRs used in the return value because these do not need to be saved.
2033 // This prevents CSR restore from clobbering return VGPRs.
2034 if (ReturnMI) {
2035 for (auto &Op : ReturnMI->operands()) {
2036 if (Op.isReg())
2037 SavedVGPRs.reset(Op.getReg());
2038 }
2039 }
2040
2041 // Create the stack objects for WWM registers now.
2042 for (Register Reg : MFI->getWWMReservedRegs()) {
2043 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
2044 MFI->allocateWWMSpill(MF, Reg, TRI->getSpillSize(*RC),
2045 TRI->getSpillAlign(*RC));
2046 }
2047
2048 // Ignore the SGPRs the default implementation found.
2049 SavedVGPRs.clearBitsNotInMask(TRI->getAllVectorRegMask());
2050
2051 // Do not save AGPRs prior to GFX90A because there was no easy way to do so.
2052 // In gfx908 there was do AGPR loads and stores and thus spilling also
2053 // require a temporary VGPR.
2054 if (!ST.hasGFX90AInsts())
2055 SavedVGPRs.clearBitsInMask(TRI->getAllAGPRRegMask());
2056
2057 determinePrologEpilogSGPRSaves(MF, SavedVGPRs, NeedExecCopyReservedReg);
2058
2059 // The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
2060 // allow the default insertion to handle them.
2061 for (auto &Reg : MFI->getWWMSpills())
2062 SavedVGPRs.reset(Reg.first);
2063}
2064
2066 BitVector &SavedRegs,
2067 RegScavenger *RS) const {
2070 if (MFI->isEntryFunction())
2071 return;
2072
2073 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2074 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2075
2076 // The SP is specifically managed and we don't want extra spills of it.
2077 SavedRegs.reset(MFI->getStackPtrOffsetReg());
2078
2079 const BitVector AllSavedRegs = SavedRegs;
2080 SavedRegs.clearBitsInMask(TRI->getAllVectorRegMask());
2081
2082 // We have to anticipate introducing CSR VGPR spills or spill of caller
2083 // save VGPR reserved for SGPR spills as we now always create stack entry
2084 // for it, if we don't have any stack objects already, since we require a FP
2085 // if there is a call and stack. We will allocate a VGPR for SGPR spills if
2086 // there are any SGPR spills. Whether they are CSR spills or otherwise.
2087 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
2088 const bool WillHaveFP =
2089 FrameInfo.hasCalls() && (AllSavedRegs.any() || MFI->hasSpilledSGPRs());
2090
2091 // FP will be specially managed like SP.
2092 if (WillHaveFP || hasFP(MF))
2093 SavedRegs.reset(MFI->getFrameOffsetReg());
2094
2095 // Return address use with return instruction is hidden through the SI_RETURN
2096 // pseudo. Given that and since the IPRA computes actual register usage and
2097 // does not use CSR list, the clobbering of return address by function calls
2098 // (D117243) or otherwise (D120922) is ignored/not seen by the IPRA's register
2099 // usage collection. This will ensure save/restore of return address happens
2100 // in those scenarios.
2101 const MachineRegisterInfo &MRI = MF.getRegInfo();
2102 Register RetAddrReg = TRI->getReturnAddressReg(MF);
2103 if (!MFI->isEntryFunction() &&
2104 (FrameInfo.hasCalls() || MRI.isPhysRegModified(RetAddrReg))) {
2105 SavedRegs.set(TRI->getSubReg(RetAddrReg, AMDGPU::sub0));
2106 SavedRegs.set(TRI->getSubReg(RetAddrReg, AMDGPU::sub1));
2107 }
2108}
2109
2111 const GCNSubtarget &ST,
2112 std::vector<CalleeSavedInfo> &CSI) {
2114 MachineFrameInfo &MFI = MF.getFrameInfo();
2115 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2116
2117 assert(
2118 llvm::is_sorted(CSI,
2119 [](const CalleeSavedInfo &A, const CalleeSavedInfo &B) {
2120 return A.getReg() < B.getReg();
2121 }) &&
2122 "Callee saved registers not sorted");
2123
2124 auto CanUseBlockOps = [&](const CalleeSavedInfo &CSI) {
2125 return !CSI.isSpilledToReg() &&
2126 TRI->getPhysRegBaseClass(CSI.getReg()) == &AMDGPU::VGPR_32RegClass &&
2127 !FuncInfo->isWWMReservedRegister(CSI.getReg());
2128 };
2129
2130 auto CSEnd = CSI.end();
2131 for (auto CSIt = CSI.begin(); CSIt != CSEnd; ++CSIt) {
2132 Register Reg = CSIt->getReg();
2133 if (!CanUseBlockOps(*CSIt))
2134 continue;
2135
2136 // Find all the regs that will fit in a 32-bit mask starting at the current
2137 // reg and build said mask. It should have 1 for every register that's
2138 // included, with the current register as the least significant bit.
2139 uint32_t Mask = 1;
2140 CSEnd = std::remove_if(
2141 CSIt + 1, CSEnd, [&](const CalleeSavedInfo &CSI) -> bool {
2142 if (CanUseBlockOps(CSI) && CSI.getReg() < Reg + 32) {
2143 Mask |= 1 << (CSI.getReg() - Reg);
2144 return true;
2145 } else {
2146 return false;
2147 }
2148 });
2149
2150 const TargetRegisterClass *BlockRegClass = TRI->getRegClassForBlockOp(MF);
2151 Register RegBlock =
2152 TRI->getMatchingSuperReg(Reg, AMDGPU::sub0, BlockRegClass);
2153 if (!RegBlock) {
2154 // We couldn't find a super register for the block. This can happen if
2155 // the register we started with is too high (e.g. v232 if the maximum is
2156 // v255). We therefore try to get the last register block and figure out
2157 // the mask from there.
2158 Register LastBlockStart =
2159 AMDGPU::VGPR0 + alignDown(Reg - AMDGPU::VGPR0, 32);
2160 RegBlock =
2161 TRI->getMatchingSuperReg(LastBlockStart, AMDGPU::sub0, BlockRegClass);
2162 assert(RegBlock && TRI->isSubRegister(RegBlock, Reg) &&
2163 "Couldn't find super register");
2164 int RegDelta = Reg - LastBlockStart;
2165 assert(RegDelta > 0 && llvm::countl_zero(Mask) >= RegDelta &&
2166 "Bad shift amount");
2167 Mask <<= RegDelta;
2168 }
2169
2170 FuncInfo->setMaskForVGPRBlockOps(RegBlock, Mask);
2171
2172 // The stack objects can be a bit smaller than the register block if we know
2173 // some of the high bits of Mask are 0. This may happen often with calling
2174 // conventions where the caller and callee-saved VGPRs are interleaved at
2175 // a small boundary (e.g. 8 or 16).
2176 int UnusedBits = llvm::countl_zero(Mask);
2177 unsigned BlockSize = TRI->getSpillSize(*BlockRegClass) - UnusedBits * 4;
2178 int FrameIdx =
2179 MFI.CreateStackObject(BlockSize, TRI->getSpillAlign(*BlockRegClass),
2180 /*isSpillSlot=*/true);
2181 MFI.setIsCalleeSavedObjectIndex(FrameIdx, true);
2182
2183 CSIt->setFrameIdx(FrameIdx);
2184 CSIt->setReg(RegBlock);
2185 }
2186 CSI.erase(CSEnd, CSI.end());
2187}
2188
2191 std::vector<CalleeSavedInfo> &CSI) const {
2192 if (CSI.empty())
2193 return true; // Early exit if no callee saved registers are modified!
2194
2195 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2196 bool UseVGPRBlocks = ST.useVGPRBlockOpsForCSR();
2197
2198 if (UseVGPRBlocks)
2199 assignSlotsUsingVGPRBlocks(MF, ST, CSI);
2200
2201 return assignCalleeSavedSpillSlotsImpl(MF, TRI, CSI) || UseVGPRBlocks;
2202}
2203
2206 std::vector<CalleeSavedInfo> &CSI) const {
2207 if (CSI.empty())
2208 return true; // Early exit if no callee saved registers are modified!
2209
2210 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
2211 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2212 const SIRegisterInfo *RI = ST.getRegisterInfo();
2213 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
2214 Register BasePtrReg = RI->getBaseRegister();
2215 Register SGPRForFPSaveRestoreCopy =
2216 FuncInfo->getScratchSGPRCopyDstReg(FramePtrReg);
2217 Register SGPRForBPSaveRestoreCopy =
2218 FuncInfo->getScratchSGPRCopyDstReg(BasePtrReg);
2219 if (!SGPRForFPSaveRestoreCopy && !SGPRForBPSaveRestoreCopy)
2220 return false;
2221
2222 unsigned NumModifiedRegs = 0;
2223
2224 if (SGPRForFPSaveRestoreCopy)
2225 NumModifiedRegs++;
2226 if (SGPRForBPSaveRestoreCopy)
2227 NumModifiedRegs++;
2228
2229 for (auto &CS : CSI) {
2230 if (CS.getReg() == FramePtrReg.asMCReg() && SGPRForFPSaveRestoreCopy) {
2231 CS.setDstReg(SGPRForFPSaveRestoreCopy);
2232 if (--NumModifiedRegs)
2233 break;
2234 } else if (CS.getReg() == BasePtrReg.asMCReg() &&
2235 SGPRForBPSaveRestoreCopy) {
2236 CS.setDstReg(SGPRForBPSaveRestoreCopy);
2237 if (--NumModifiedRegs)
2238 break;
2239 }
2240 }
2241
2242 return false;
2243}
2244
2246 const MachineFunction &MF) const {
2247
2248 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2249 const MachineFrameInfo &MFI = MF.getFrameInfo();
2250 const SIInstrInfo *TII = ST.getInstrInfo();
2251 uint64_t EstStackSize = MFI.estimateStackSize(MF);
2252 uint64_t MaxOffset = EstStackSize - 1;
2253
2254 // We need the emergency stack slots to be allocated in range of the
2255 // MUBUF/flat scratch immediate offset from the base register, so assign these
2256 // first at the incoming SP position.
2257 //
2258 // TODO: We could try sorting the objects to find a hole in the first bytes
2259 // rather than allocating as close to possible. This could save a lot of space
2260 // on frames with alignment requirements.
2261 if (ST.hasFlatScratchEnabled()) {
2262 if (TII->isLegalFLATOffset(MaxOffset, AMDGPUAS::PRIVATE_ADDRESS,
2264 return false;
2265 } else {
2266 if (TII->isLegalMUBUFImmOffset(MaxOffset))
2267 return false;
2268 }
2269
2270 return true;
2271}
2272
2273/// Return the set of all root registers of regunits live-in to @p MBB.
2274///
2275/// Intended to avoid using the expensive @c MCRegAliasIterator when deciding
2276/// if a register to be spilled is already live-in (see @c isAnyRootLiveIn).
2278 const SIRegisterInfo &TRI) {
2279 SparseBitVector<> LiveInRoots;
2280 for (const auto &LI : MBB.liveins()) {
2281 for (MCRegUnitMaskIterator MI(LI.PhysReg, &TRI); MI.isValid(); ++MI) {
2282 auto [Unit, UnitLaneMask] = *MI;
2283 if ((LI.LaneMask & UnitLaneMask).none())
2284 continue;
2285 for (MCRegUnitRootIterator RI(Unit, &TRI); RI.isValid(); ++RI)
2286 LiveInRoots.set(*RI);
2287 }
2288 }
2289 return LiveInRoots;
2290}
2291
2292/// Returns true iff any root of @p Reg is in @p LiveInRoots
2293/// (see @c buildLiveInRoots).
2294static bool isAnyRootLiveIn(const SparseBitVector<> &LiveInRoots,
2295 const SIRegisterInfo &TRI, MCRegister Reg) {
2296 for (MCRegUnitIterator UI(Reg, &TRI); UI.isValid(); ++UI) {
2297 for (MCRegUnitRootIterator RI(*UI, &TRI); RI.isValid(); ++RI) {
2298 if (LiveInRoots.test(*RI))
2299 return true;
2300 }
2301 }
2302 return false;
2303}
2304
2305void SIFrameLowering::spillCalleeSavedRegisterWithoutBlockOps(
2307 const CalleeSavedInfo &CS, const SIInstrInfo *TII,
2308 const SIRegisterInfo &TRI,
2309 const std::optional<SparseBitVector<>> &LiveInRoots) const {
2310 MCRegister Reg = CS.getReg();
2311
2312 // We assume a sortUniqueLiveIns later
2313 MBB.addLiveIn(Reg);
2314
2315 if (CS.isSpilledToReg()) {
2316 BuildMI(MBB, MI, DebugLoc(), TII->get(TargetOpcode::COPY), CS.getDstReg())
2317 .addReg(Reg, getKillRegState(true));
2318 } else {
2319 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
2320 bool IsKill = true;
2321 // If this value was already livein, we probably have a direct use of
2322 // the incoming register value, so don't kill at the spill point. This
2323 // happens since we pass some special inputs (workgroup IDs) in the
2324 // callee saved range.
2325 if (LiveInRoots)
2326 IsKill = !isAnyRootLiveIn(*LiveInRoots, TRI, Reg);
2327 TII->storeRegToStackSlotCFI(MBB, MI, Reg, IsKill, CS.getFrameIdx(), RC);
2328 }
2329}
2330
2333 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *OrigTRI) const {
2334 auto &TRI = *static_cast<const SIRegisterInfo *>(OrigTRI);
2335 MachineFunction *MF = MBB.getParent();
2336 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2337 const SIInstrInfo *TII = ST.getInstrInfo();
2338
2339 std::optional<SparseBitVector<>> LiveInRoots;
2340 if (MBB.getParent()->getRegInfo().tracksLiveness())
2341 LiveInRoots = buildLiveInRoots(MBB, TRI);
2342
2343 if (!ST.useVGPRBlockOpsForCSR()) {
2344 for (const CalleeSavedInfo &CS : CSI)
2345 spillCalleeSavedRegisterWithoutBlockOps(MBB, MI, CS, TII, TRI,
2346 LiveInRoots);
2347 if (LiveInRoots)
2348 MBB.sortUniqueLiveIns();
2349 return true;
2350 }
2351
2352 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
2354
2355 const TargetRegisterClass *BlockRegClass = TRI.getRegClassForBlockOp(*MF);
2356 for (const CalleeSavedInfo &CS : CSI) {
2357 Register Reg = CS.getReg();
2358 if (!BlockRegClass->contains(Reg) ||
2359 !FuncInfo->hasMaskForVGPRBlockOps(Reg)) {
2360 spillCalleeSavedRegisterWithoutBlockOps(MBB, MI, CS, TII, TRI,
2361 LiveInRoots);
2362 continue;
2363 }
2364
2365 // Build a scratch block store.
2366 uint32_t Mask = FuncInfo->getMaskForVGPRBlockOps(Reg);
2367 int FrameIndex = CS.getFrameIdx();
2368 MachinePointerInfo PtrInfo =
2369 MachinePointerInfo::getFixedStack(*MF, FrameIndex);
2370 MachineMemOperand *MMO =
2372 FrameInfo.getObjectSize(FrameIndex),
2373 FrameInfo.getObjectAlign(FrameIndex));
2374
2375 BuildMI(MBB, MI, MI->getDebugLoc(),
2376 TII->get(AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE))
2377 .addReg(Reg, getKillRegState(false))
2378 .addFrameIndex(FrameIndex)
2379 .addReg(FuncInfo->getStackPtrOffsetReg())
2380 .addImm(0)
2381 .addImm(Mask)
2382 .addMemOperand(MMO);
2383
2384 FuncInfo->setHasSpilledVGPRs();
2385
2386 // Add the register to the liveins. This is necessary because if any of the
2387 // VGPRs in the register block is reserved (e.g. if it's a WWM register),
2388 // then the whole block will be marked as reserved and `updateLiveness` will
2389 // skip it.
2390 if (LiveInRoots)
2391 MBB.addLiveIn(Reg);
2392 }
2393 if (LiveInRoots)
2394 MBB.sortUniqueLiveIns();
2395
2396 return true;
2397}
2398
2402 const TargetRegisterInfo *OrigTRI) const {
2403 auto &TRI = *static_cast<const SIRegisterInfo *>(OrigTRI);
2404 MachineFunction *MF = MBB.getParent();
2405 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2406 if (!ST.useVGPRBlockOpsForCSR())
2407 return false;
2408
2410 MachineFrameInfo &MFI = MF->getFrameInfo();
2411 const SIInstrInfo *TII = ST.getInstrInfo();
2412 const TargetRegisterClass *BlockRegClass = TRI.getRegClassForBlockOp(*MF);
2413 for (const CalleeSavedInfo &CS : reverse(CSI)) {
2414 Register Reg = CS.getReg();
2415 if (!BlockRegClass->contains(Reg) ||
2416 !FuncInfo->hasMaskForVGPRBlockOps(Reg)) {
2418 continue;
2419 }
2420
2421 // Build a scratch block load.
2422 uint32_t Mask = FuncInfo->getMaskForVGPRBlockOps(Reg);
2423 int FrameIndex = CS.getFrameIdx();
2424 MachinePointerInfo PtrInfo =
2425 MachinePointerInfo::getFixedStack(*MF, FrameIndex);
2427 PtrInfo, MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIndex),
2428 MFI.getObjectAlign(FrameIndex));
2429
2430 auto MIB = BuildMI(MBB, MI, MI->getDebugLoc(),
2431 TII->get(AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE), Reg)
2432 .addFrameIndex(FrameIndex)
2433 .addReg(FuncInfo->getStackPtrOffsetReg())
2434 .addImm(0)
2435 .addImm(Mask)
2436 .addMemOperand(MMO);
2437 TRI.addImplicitUsesForBlockCSRLoad(MIB, Reg);
2438
2439 // Add the register to the liveins. This is necessary because if any of the
2440 // VGPRs in the register block is reserved (e.g. if it's a WWM register),
2441 // then the whole block will be marked as reserved and `updateLiveness` will
2442 // skip it.
2443 MBB.addLiveIn(Reg);
2444 }
2445
2446 MBB.sortUniqueLiveIns();
2447 return true;
2448}
2449
2451 MachineFunction &MF,
2454 int64_t Amount = I->getOperand(0).getImm();
2455 if (Amount == 0)
2456 return MBB.erase(I);
2457
2458 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2459 const SIInstrInfo *TII = ST.getInstrInfo();
2460 const DebugLoc &DL = I->getDebugLoc();
2461 unsigned Opc = I->getOpcode();
2462 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
2463 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
2464
2465 if (!hasReservedCallFrame(MF)) {
2466 Amount = alignTo(Amount, getStackAlign());
2467 assert(isUInt<32>(Amount) && "exceeded stack address space size");
2470
2471 Amount *= getScratchScaleFactor(ST);
2472 if (IsDestroy)
2473 Amount = -Amount;
2474 auto Add = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SPReg)
2475 .addReg(SPReg)
2476 .addImm(Amount);
2477 Add->getOperand(3).setIsDead(); // Mark SCC as dead.
2478 } else if (CalleePopAmount != 0) {
2479 llvm_unreachable("is this used?");
2480 }
2481
2482 return MBB.erase(I);
2483}
2484
2485/// Returns true if the frame will require a reference to the stack pointer.
2486///
2487/// This is the set of conditions common to setting up the stack pointer in a
2488/// kernel, and for using a frame pointer in a callable function.
2489///
2490/// FIXME: Should also check hasOpaqueSPAdjustment and if any inline asm
2491/// references SP.
2493 return MFI.hasVarSizedObjects() || MFI.hasStackMap() || MFI.hasPatchPoint();
2494}
2495
2496// The FP for kernels is always known 0, so we never really need to setup an
2497// explicit register for it. However, DisableFramePointerElim will force us to
2498// use a register for it.
2500 const MachineFrameInfo &MFI = MF.getFrameInfo();
2501
2502 // For entry functions we can use an immediate offset in most cases,
2503 // so the presence of calls doesn't imply we need a distinct frame pointer.
2504 if (MFI.hasCalls() &&
2506 // All offsets are unsigned, so need to be addressed in the same direction
2507 // as stack growth.
2508
2509 // FIXME: This function is pretty broken, since it can be called before the
2510 // frame layout is determined or CSR spills are inserted.
2511 return MFI.getStackSize() != 0;
2512 }
2513
2514 return frameTriviallyRequiresSP(MFI) || MFI.isFrameAddressTaken() ||
2515 MF.getSubtarget<GCNSubtarget>().getRegisterInfo()->hasStackRealignment(
2516 MF) ||
2518}
2519
2521 const MachineFunction &MF) const {
2522 return MF.getInfo<SIMachineFunctionInfo>()->isDynamicVGPREnabled() &&
2525}
2526
2527// This is essentially a reduced version of hasFP for entry functions. Since the
2528// stack pointer is known 0 on entry to kernels, we never really need an FP
2529// register. We may need to initialize the stack pointer depending on the frame
2530// properties, which logically overlaps many of the cases where an ordinary
2531// function would require an FP.
2533 const MachineFunction &MF) const {
2534 // Callable functions always require a stack pointer reference.
2536 "only expected to call this for entry points functions");
2537
2538 const MachineFrameInfo &MFI = MF.getFrameInfo();
2539
2540 // Entry points ordinarily don't need to initialize SP. We have to set it up
2541 // for callees if there are any.
2542 if (MFI.hasCalls())
2543 return true;
2544
2545 // We still need to initialize the SP if we're doing anything weird that
2546 // references the SP, like variable sized stack objects.
2547 return frameTriviallyRequiresSP(MFI);
2548}
2549
2552 const DebugLoc &DL,
2553 const MCCFIInstruction &CFIInst,
2554 MachineInstr::MIFlag Flag) const {
2555 MachineFunction &MF = *MBB.getParent();
2556 const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
2557 return BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
2558 .addCFIIndex(MF.addFrameInst(CFIInst))
2559 .setMIFlag(Flag);
2560}
2561
2564 const DebugLoc &DL, const MCRegister Reg, const MCRegister RegCopy) const {
2565 MachineFunction &MF = *MBB.getParent();
2566 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2567 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2568
2569 unsigned MaskReg = MCRI.getDwarfRegNum(
2570 ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC, false);
2572 nullptr, MCRI.getDwarfRegNum(Reg, false),
2573 MCRI.getDwarfRegNum(RegCopy, false), VGPRLaneBitSize, MaskReg,
2574 ST.getWavefrontSize());
2575 return buildCFI(MBB, MBBI, DL, std::move(CFIInst));
2576}
2577
2580 const DebugLoc &DL, const MCRegister SGPR, const MCRegister VGPR,
2581 const int Lane) const {
2582 const MachineFunction &MF = *MBB.getParent();
2583 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2584
2585 int DwarfSGPR = MCRI.getDwarfRegNum(SGPR, false);
2586 int DwarfVGPR = MCRI.getDwarfRegNum(VGPR, false);
2587 assert(DwarfSGPR != -1 && DwarfVGPR != -1);
2588 assert(Lane != -1 && "Expected a lane to be present");
2589
2590 // Build a CFI instruction that represents a SGPR spilled to a single lane of
2591 // a VGPR.
2593 unsigned(Lane), VGPRLaneBitSize};
2594 auto CFIInst =
2595 MCCFIInstruction::createLLVMVectorRegisters(nullptr, DwarfSGPR, {VR});
2596 return buildCFI(MBB, MBBI, DL, std::move(CFIInst));
2597}
2598
2601 const DebugLoc &DL, MCRegister SGPR,
2602 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills) const {
2603 if (VGPRSpills.size() == 1u)
2604 return buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, SGPR, VGPRSpills[0].VGPR,
2605 VGPRSpills[0].Lane);
2606 const MachineFunction &MF = *MBB.getParent();
2607 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2608
2609 int DwarfSGPR = MCRI.getDwarfRegNum(SGPR, false);
2610 assert(DwarfSGPR != -1);
2611
2612 // Build a CFI instruction that represents a SGPR spilled to multiple lanes of
2613 // multiple VGPRs.
2614
2616 for (SIRegisterInfo::SpilledReg Spill : VGPRSpills) {
2617 int DwarfVGPR = MCRI.getDwarfRegNum(Spill.VGPR, false);
2618 assert(DwarfVGPR != -1);
2619 assert(Spill.hasLane() && "Expected a lane to be present");
2620 VGPRs.push_back(
2621 {unsigned(DwarfVGPR), unsigned(Spill.Lane), VGPRLaneBitSize});
2622 }
2623
2624 auto CFIInst = MCCFIInstruction::createLLVMVectorRegisters(nullptr, DwarfSGPR,
2625 std::move(VGPRs));
2626 return buildCFI(MBB, MBBI, DL, std::move(CFIInst));
2627}
2628
2631 const DebugLoc &DL, MCRegister SGPR, int64_t Offset) const {
2632 MachineFunction &MF = *MBB.getParent();
2633 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2634 return buildCFI(MBB, MBBI, DL,
2636 nullptr, MCRI.getDwarfRegNum(SGPR, false), Offset));
2637}
2638
2641 const DebugLoc &DL, MCRegister VGPR, int64_t Offset) const {
2642 const MachineFunction &MF = *MBB.getParent();
2643 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2644 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2645
2646 int DwarfVGPR = MCRI.getDwarfRegNum(VGPR, false);
2647 assert(DwarfVGPR != -1);
2648
2649 unsigned MaskReg = MCRI.getDwarfRegNum(
2650 ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC, false);
2652 nullptr, DwarfVGPR, VGPRLaneBitSize, MaskReg, ST.getWavefrontSize(),
2653 Offset);
2654 return buildCFI(MBB, MBBI, DL, std::move(CFIInst));
2655}
2656
2658 SmallVectorImpl<unsigned> &DwarfRegs) {
2659 int64_t DwarfReg = TRI.getDwarfRegNum(Reg, /*IsEH=*/false);
2660 if (DwarfReg >= 0) {
2661 DwarfRegs.push_back(unsigned(DwarfReg));
2662 return;
2663 }
2664
2665 const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg);
2666 ArrayRef<int16_t> SplitParts = TRI.getRegSplitParts(RC, SGPRByteSize);
2667 for (unsigned I = 0; I < SplitParts.size(); ++I) {
2668 MCRegister SubReg = TRI.getSubReg(Reg, SplitParts[I]);
2669 DwarfReg = TRI.getDwarfRegNum(SubReg, /*IsEH=*/false);
2670 if (DwarfReg < 0) {
2671 DwarfRegs.clear();
2672 return;
2673 }
2674 DwarfRegs.push_back(unsigned(DwarfReg));
2675 }
2676}
2677
2680 const DebugLoc &DL, MCRegister Reg, MCRegister CopyReg) const {
2681 const MachineFunction &MF = *MBB.getParent();
2682 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2683 const SIRegisterInfo &TRI = *ST.getRegisterInfo();
2684
2685 SmallVector<unsigned> DwarfRegs, DwarfCopyRegs;
2686 decomposeDwarfRegs(TRI, Reg, DwarfRegs);
2687 decomposeDwarfRegs(TRI, CopyReg, DwarfCopyRegs);
2688
2689 if (DwarfRegs.size() == DwarfCopyRegs.size()) {
2690 for (unsigned I = 0, E = DwarfRegs.size(); I < E; ++I) {
2691 auto CFI = MCCFIInstruction::createRegister(nullptr, DwarfRegs[I],
2692 DwarfCopyRegs[I]);
2693 buildCFI(MBB, MBBI, DL, CFI);
2694 }
2695 return;
2696 }
2697
2698 if (DwarfRegs.size() == 1 && DwarfCopyRegs.size() == 2) {
2700 nullptr, DwarfRegs[0], DwarfCopyRegs[0], SGPRBitSize, DwarfCopyRegs[1],
2701 SGPRBitSize);
2702 buildCFI(MBB, MBBI, DL, CFI);
2703 return;
2704 }
2705
2706 for (unsigned I = 0, E = DwarfRegs.size(); I < E; ++I) {
2707 auto CFI = MCCFIInstruction::createUndefined(nullptr, DwarfRegs[I]);
2708 buildCFI(MBB, MBBI, DL, CFI);
2709 }
2710}
2711
2714 const DebugLoc &DL, MCRegister SGPR, MCRegister VCCHalf) const {
2715 const MachineFunction &MF = *MBB.getParent();
2716 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2718
2719 int DwarfSGPR = MCRI.getDwarfRegNum(SGPR, false);
2720 int DwarfVCC = MCRI.getDwarfRegNum(AMDGPU::VCC, false);
2721 assert(DwarfSGPR != -1 && DwarfVCC != -1);
2722
2723 // Describe the copy as one 32-bit lane of VCC_64: lane 0 is vcc_lo and lane 1
2724 // is vcc_hi.
2726 unsigned(DwarfVCC), VCCHalf == AMDGPU::VCC_HI ? 1u : 0u, SGPRBitSize};
2727 auto CFIInst =
2728 MCCFIInstruction::createLLVMVectorRegisters(nullptr, DwarfSGPR, {Half});
2729 return buildCFI(MBB, MBBI, DL, std::move(CFIInst));
2730}
2731
2734 const DebugLoc &DL, MCRegister Reg) const {
2735 const MachineFunction &MF = *MBB.getParent();
2736 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2737 int DwarfReg = MCRI.getDwarfRegNum(Reg, /*isEH=*/false);
2738 auto CFIInst = MCCFIInstruction::createSameValue(nullptr, DwarfReg);
2739 return buildCFI(MBB, MBBI, DL, std::move(CFIInst));
2740}
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
Provides AMDGPU specific target descriptions.
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains constants used for implementing Dwarf debug support.
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
A set of register units.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static constexpr MCPhysReg FPReg
static constexpr MCPhysReg SPReg
MachineRegisterInfo & MRI
This file declares the machine register scavenger class.
static void buildEpilogRestore(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SpillReg, int FI, Register FrameReg, int64_t DwordOff=0)
static void getVGPRSpillLaneOrTempRegister(MachineFunction &MF, LiveRegUnits &LiveUnits, Register SGPR, const TargetRegisterClass &RC=AMDGPU::SReg_32_XM0_XEXECRegClass, bool IncludeScratchCopy=true)
Query target location for spilling SGPRs IncludeScratchCopy : Also look for free scratch SGPRs.
static void buildGitPtr(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, const SIInstrInfo *TII, Register TargetReg)
static bool allStackObjectsAreDead(const MachineFrameInfo &MFI)
static MCCFIInstruction createScaledCFAInPrivateWave(const GCNSubtarget &ST, int64_t DwarfStackPtrReg)
static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SpillReg, int FI, Register FrameReg, int64_t DwordOff=0)
static Register buildScratchExecCopy(LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool IsProlog, bool EnableInactiveLanes)
static void encodeDwarfRegisterLocation(int DwarfReg, raw_ostream &OS)
static constexpr unsigned SGPRBitSize
static bool frameTriviallyRequiresSP(const MachineFrameInfo &MFI)
Returns true if the frame will require a reference to the stack pointer.
static void decomposeDwarfRegs(const SIRegisterInfo &TRI, MCRegister Reg, SmallVectorImpl< unsigned > &DwarfRegs)
static SparseBitVector buildLiveInRoots(const MachineBasicBlock &MBB, const SIRegisterInfo &TRI)
Return the set of all root registers of regunits live-in to MBB.
static void initLiveUnits(LiveRegUnits &LiveUnits, const SIRegisterInfo &TRI, const SIMachineFunctionInfo *FuncInfo, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsProlog)
static constexpr unsigned VGPRLaneBitSize
static bool allSGPRSpillsAreDead(const MachineFunction &MF)
static bool isWave64VCCHalf(const GCNSubtarget &ST, MCRegister Reg)
static MCRegister findScratchNonCalleeSaveRegister(MachineRegisterInfo &MRI, LiveRegUnits &LiveUnits, const TargetRegisterClass &RC, bool Unused=false)
static MCRegister findUnusedRegister(MachineRegisterInfo &MRI, const LiveRegUnits &LiveUnits, const TargetRegisterClass &RC)
static constexpr unsigned SGPRByteSize
static void assignSlotsUsingVGPRBlocks(MachineFunction &MF, const GCNSubtarget &ST, std::vector< CalleeSavedInfo > &CSI)
static bool isAnyRootLiveIn(const SparseBitVector<> &LiveInRoots, const SIRegisterInfo &TRI, MCRegister Reg)
Returns true iff any root of Reg is in LiveInRoots (see buildLiveInRoots).
static unsigned getScratchScaleFactor(const GCNSubtarget &ST)
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
#define LLVM_DEBUG(...)
Definition Debug.h:119
static const int BlockSize
Definition TarWriter.cpp:33
static const LaneMaskConstants & get(const GCNSubtarget &ST)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition ArrayRef.h:185
bool test(unsigned Idx) const
Returns true if bit Idx is set.
Definition BitVector.h:483
BitVector & reset()
Reset all bits in the bitvector.
Definition BitVector.h:410
void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
Clear a bit in this vector for every '0' bit in Mask.
Definition BitVector.h:761
BitVector & set()
Set all bits in the bitvector.
Definition BitVector.h:367
bool any() const
Returns true if any bit is set.
Definition BitVector.h:190
void clearBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
Clear any bits in this vector that are set in Mask.
Definition BitVector.h:749
iterator_range< const_set_bits_iterator > set_bits() const
Definition BitVector.h:160
bool empty() const
Returns whether there are no bits in this bitvector.
Definition BitVector.h:176
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
MCRegister getReg() const
MCRegister getDstReg() const
A debug info location.
Definition DebugLoc.h:126
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
const HexagonRegisterInfo & getRegisterInfo() const
A set of register units used to track register liveness.
bool available(MCRegister Reg) const
Returns true if no part of physical register Reg is live.
void init(const TargetRegisterInfo &TRI)
Initialize and clear the set.
void addReg(MCRegister Reg)
Adds register units covered by physical register Reg.
LLVM_ABI void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
LLVM_ABI void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
void removeReg(MCRegister Reg)
Removes all register units covered by physical register Reg.
bool empty() const
Returns true if the set is empty.
LLVM_ABI void addLiveIns(const MachineBasicBlock &MBB)
Adds registers living into block MBB.
static MCCFIInstruction createLLVMVectorOffset(MCSymbol *L, unsigned Register, unsigned RegisterSizeInBits, unsigned MaskRegister, unsigned MaskRegisterSizeInBits, int64_t Offset, SMLoc Loc={})
.cfi_llvm_vector_offset Previous value of Register is saved at Offset from CFA.
Definition MCDwarf.h:797
static MCCFIInstruction createUndefined(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_undefined From now on the previous value of Register can't be restored anymore.
Definition MCDwarf.h:732
static MCCFIInstruction createLLVMVectorRegisters(MCSymbol *L, unsigned Register, ArrayRef< VectorRegisterWithLane > VectorRegisters, SMLoc Loc={})
.cfi_llvm_vector_registers Previous value of Register is saved in lanes of vector registers.
Definition MCDwarf.h:787
static MCCFIInstruction createLLVMVectorRegisterMask(MCSymbol *L, unsigned Register, unsigned SpillRegister, unsigned SpillRegisterLaneSizeInBits, unsigned MaskRegister, unsigned MaskRegisterSizeInBits, SMLoc Loc={})
.cfi_llvm_vector_register_mask Previous value of Register is saved in SpillRegister,...
Definition MCDwarf.h:808
static MCCFIInstruction createRegister(MCSymbol *L, unsigned Register1, unsigned Register2, SMLoc Loc={})
.cfi_register Previous value of Register1 is saved in register Register2.
Definition MCDwarf.h:685
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
Definition MCDwarf.h:670
static MCCFIInstruction createLLVMRegisterPair(MCSymbol *L, unsigned Register, unsigned R1, unsigned R1SizeInBits, unsigned R2, unsigned R2SizeInBits, SMLoc Loc={})
.cfi_llvm_register_pair Previous value of Register is saved in R1:R2.
Definition MCDwarf.h:777
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition MCDwarf.h:756
static MCCFIInstruction createSameValue(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_same_value Current value of Register is the same as in the previous frame.
Definition MCDwarf.h:739
const MCRegisterInfo * getRegisterInfo() const
Definition MCContext.h:413
Describe properties that are true of each instruction in the target description file.
bool isValid() const
Returns true if this iterator is not yet at the end.
MCRegUnitMaskIterator enumerates a list of register units and their associated lane masks for Reg.
MCRegUnitRootIterator enumerates the root registers of a register unit.
bool isValid() const
Check if the iterator is at the end of the list.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
virtual int64_t getDwarfRegNum(MCRegister Reg, bool isEH) const
Map a target register to an equivalent dwarf register number.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition MCRegister.h:72
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
bool hasCalls() const
Return true if the current function has any function calls.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Align getMaxAlign() const
Return alignment of this function's frame.
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
bool hasTailCall() const
Returns true if the function contains a tail call.
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
LLVM_ABI int CreateSpillStackObject(uint64_t Size, Align Alignment, TargetStackID::Value StackID=TargetStackID::Default)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getObjectIndexBegin() const
Return the minimum frame object index.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
bool needsFrameMoves() const
True if this function needs frame moves for debug or exceptions.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
bool disableFramePointerElim() const
Returns true if frame pointer elimination should be disabled for this function.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
mop_range operands()
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
void setIsDead(bool Val=true)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool isReserved(MCRegister PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
bool isAllocatable(MCRegister PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn't been...
LLVM_ABI const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
void reserveReg(MCRegister PhysReg, const TargetRegisterInfo *TRI)
reserveReg – Mark a register as reserved so checks like isAllocatable will not suggest using it.
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
LLVM_ABI bool isPhysRegModified(MCRegister PhysReg, bool SkipNoReturnDef=false) const
Return true if the specified register is modified in this function.
LLVM_ABI bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
PrologEpilogSGPRSpillBuilder(Register Reg, const PrologEpilogSGPRSaveRestoreInfo SI, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, const SIInstrInfo *TII, const SIRegisterInfo &TRI, LiveRegUnits &LiveUnits, Register FrameReg, bool IsFramePtrPrologSpill=false)
Wrapper class representing virtual and physical registers.
Definition Register.h:20
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition Register.h:107
void determinePrologEpilogSGPRSaves(MachineFunction &MF, BitVector &SavedRegs, bool NeedExecCopyReservedReg) const
MachineInstr * buildCFIForSGPRToVMEMSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister SGPR, int64_t Offset) const
Create a CFI index describing a spill of a SGPR to VMEM and build a MachineInstr around it.
void buildCFIForSRegToSRegSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister Reg, MCRegister CopyReg) const
void emitCSRSpillRestores(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy) const
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - This method should return the base register and offset used to reference a f...
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS=nullptr) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
bool mayReserveScratchForCWSR(const MachineFunction &MF) const
bool allocateScavengingFrameIndexesNearIncomingSP(const MachineFunction &MF) const override
Control the placement of special register scavenging spill slots when allocating a stack frame.
bool requiresStackPointerReference(const MachineFunction &MF) const
void emitEntryFunctionPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const
MachineInstr * buildCFIForSGPRToVCCHalfCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister SGPR, MCRegister VCCHalf) const
Create a CFI index describing a copy of the 32-bit SGPR to VCCHalf (vcc_lo or vcc_hi) in wave64 and b...
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
bool hasFPImpl(const MachineFunction &MF) const override
bool assignCalleeSavedSpillSlotsImpl(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI) const
MachineInstr * buildCFIForVRegToVRegSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCRegister Reg, const MCRegister RegCopy) const
Create a CFI index describing a spill of the VGPR/AGPR Reg to another VGPR/AGPR RegCopy and build a M...
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
MachineInstr * buildCFIForVGPRToVMEMSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister VGPR, int64_t Offset) const
Create a CFI index describing a spill of a VGPR to VMEM and build a MachineInstr around it.
void emitEntryFunctionEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const
MachineInstr * buildCFIForSGPRToVGPRSpill(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCRegister SGPR, const MCRegister VGPR, const int Lane) const
Create a CFI index describing a spill of an SGPR to a single lane of a VGPR and build a MachineInstr ...
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
void determineCalleeSavesSGPR(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
MachineInstr * buildCFIForSameValue(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister Reg) const
MachineInstr * buildCFI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const MCCFIInstruction &CFIInst, MachineInstr::MIFlag flag=MachineInstr::FrameSetup) const
Create a CFI index for CFIInst and build a MachineInstr around it.
void emitCSRSpillStores(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy, const bool NeedsFrameMoves) const
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS=nullptr) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
bool isSupportedStackID(TargetStackID::Value ID) const override
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
ArrayRef< PrologEpilogSGPRSpill > getPrologEpilogSGPRSpills() const
const WWMSpillsMap & getWWMSpills() const
void getAllScratchSGPRCopyDstRegs(SmallVectorImpl< Register > &Regs) const
ArrayRef< MCPhysReg > getAGPRSpillVGPRs() const
void removePrologEpilogSGPRSpillEntry(Register Reg)
void shiftWwmVGPRsToLowestRange(MachineFunction &MF, SmallVectorImpl< Register > &WWMVGPRs, BitVector &SavedVGPRs)
void setMaskForVGPRBlockOps(Register RegisterBlock, uint32_t Mask)
GCNUserSGPRUsageInfo & getUserSGPRInfo()
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
void setVGPRToAGPRSpillDead(int FrameIndex)
Register getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses.
ArrayRef< MCPhysReg > getVGPRSpillAGPRs() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
uint32_t getMaskForVGPRBlockOps(Register RegisterBlock) const
bool hasMaskForVGPRBlockOps(Register RegisterBlock) const
bool hasPrologEpilogSGPRSpillEntry(Register Reg) const
Register getGITPtrLoReg(const MachineFunction &MF) const
void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy)
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
bool isWWMReservedRegister(Register Reg) const
ArrayRef< SIRegisterInfo::SpilledReg > getSGPRSpillToPhysicalVGPRLanes(int FrameIndex) const
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool SpillToPhysVGPRLane=false, bool IsPrologEpilog=false)
void setLongBranchReservedReg(Register Reg)
void setHasSpilledVGPRs(bool Spill=true)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
void setScratchReservedForDynamicVGPRs(unsigned SizeInBytes)
MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
const ReservedRegSet & getWWMReservedRegs() const
const PrologEpilogSGPRSaveRestoreInfo & getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const
void setIsStackRealigned(bool Realigned=true)
void addToPrologEpilogSGPRSpills(Register Reg, PrologEpilogSGPRSaveRestoreInfo SI)
Register getScratchSGPRCopyDstReg(Register Reg) const
Register getFrameRegister(const MachineFunction &MF) const override
Represents a location in source code.
Definition SMLoc.h:22
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:28
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
void set(unsigned Idx)
bool test(unsigned Idx) const
StackOffset holds a fixed and a scalable offset in bytes.
Definition TypeSize.h:31
int64_t getFixed() const
Returns the fixed component of the stack.
Definition TypeSize.h:47
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
virtual bool hasReservedCallFrame(const MachineFunction &MF) const
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
void restoreCalleeSavedRegister(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const CalleeSavedInfo &CS, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an SmallVector or SmallString.
StringRef str() const
Return a StringRef for the vector contents.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ PRIVATE_ADDRESS
Address space for private memory.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI, unsigned DynamicVGPRBlockSize, std::optional< bool > EnableWavefrontSize32)
uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset)
Convert ByteOffset to dwords if the subtarget uses dword SMRD immediate offsets.
LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isCompute(CallingConv::ID CC)
LLVM_READNONE constexpr bool isChainCC(CallingConv::ID CC)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1748
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
constexpr RegState getKillRegState(bool B)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
void clearDebugInfoForSpillFIs(MachineFrameInfo &MFI, MachineBasicBlock &MBB, const BitVector &SpillFIs)
Replace frame index operands with null registers in debug value instructions for the specified spill ...
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
auto make_first_range(ContainerTy &&c)
Given a container of pairs, return a range over the first elements.
Definition STLExtras.h:1415
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
bool is_sorted(R &&Range, Compare C)
Wrapper function around std::is_sorted to check if elements in a range R are sorted with respect to a...
Definition STLExtras.h:1986
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:2035
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition LEB128.h:79
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.