LLVM  9.0.0svn
SIFrameLowering.cpp
Go to the documentation of this file.
1 //===----------------------- SIFrameLowering.cpp --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 
9 #include "SIFrameLowering.h"
10 #include "AMDGPUSubtarget.h"
11 #include "SIInstrInfo.h"
12 #include "SIMachineFunctionInfo.h"
13 #include "SIRegisterInfo.h"
15 
21 
22 using namespace llvm;
23 
24 
26  const MachineFunction &MF) {
27  return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(),
28  ST.getMaxNumSGPRs(MF) / 4);
29 }
30 
32  const MachineFunction &MF) {
33  return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(),
34  ST.getMaxNumSGPRs(MF));
35 }
36 
37 void SIFrameLowering::emitFlatScratchInit(const GCNSubtarget &ST,
38  MachineFunction &MF,
39  MachineBasicBlock &MBB) const {
40  const SIInstrInfo *TII = ST.getInstrInfo();
41  const SIRegisterInfo* TRI = &TII->getRegisterInfo();
43 
44  // We don't need this if we only have spills since there is no user facing
45  // scratch.
46 
47  // TODO: If we know we don't have flat instructions earlier, we can omit
48  // this from the input registers.
49  //
50  // TODO: We only need to know if we access scratch space through a flat
51  // pointer. Because we only detect if flat instructions are used at all,
52  // this will be used more often than necessary on VI.
53 
54  // Debug location must be unknown since the first debug location is used to
55  // determine the end of the prologue.
56  DebugLoc DL;
58 
59  unsigned FlatScratchInitReg
61 
63  MRI.addLiveIn(FlatScratchInitReg);
64  MBB.addLiveIn(FlatScratchInitReg);
65 
66  unsigned FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0);
67  unsigned FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1);
68 
69  unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg();
70 
71  // Do a 64-bit pointer add.
72  if (ST.flatScratchIsPointer()) {
73  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), AMDGPU::FLAT_SCR_LO)
74  .addReg(FlatScrInitLo)
75  .addReg(ScratchWaveOffsetReg);
76  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), AMDGPU::FLAT_SCR_HI)
77  .addReg(FlatScrInitHi)
78  .addImm(0);
79 
80  return;
81  }
82 
83  // Copy the size in bytes.
84  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::FLAT_SCR_LO)
85  .addReg(FlatScrInitHi, RegState::Kill);
86 
87  // Add wave offset in bytes to private base offset.
88  // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init.
89  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo)
90  .addReg(FlatScrInitLo)
91  .addReg(ScratchWaveOffsetReg);
92 
93  // Convert offset to 256-byte units.
94  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI)
95  .addReg(FlatScrInitLo, RegState::Kill)
96  .addImm(8);
97 }
98 
99 unsigned SIFrameLowering::getReservedPrivateSegmentBufferReg(
100  const GCNSubtarget &ST,
101  const SIInstrInfo *TII,
102  const SIRegisterInfo *TRI,
104  MachineFunction &MF) const {
106 
107  // We need to insert initialization of the scratch resource descriptor.
108  unsigned ScratchRsrcReg = MFI->getScratchRSrcReg();
109  if (ScratchRsrcReg == AMDGPU::NoRegister ||
110  !MRI.isPhysRegUsed(ScratchRsrcReg))
111  return AMDGPU::NoRegister;
112 
113  if (ST.hasSGPRInitBug() ||
114  ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
115  return ScratchRsrcReg;
116 
117  // We reserved the last registers for this. Shift it down to the end of those
118  // which were actually used.
119  //
120  // FIXME: It might be safer to use a pseudoregister before replacement.
121 
122  // FIXME: We should be able to eliminate unused input registers. We only
123  // cannot do this for the resources required for scratch access. For now we
124  // skip over user SGPRs and may leave unused holes.
125 
126  // We find the resource first because it has an alignment requirement.
127 
128  unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4;
129  ArrayRef<MCPhysReg> AllSGPR128s = getAllSGPR128(ST, MF);
130  AllSGPR128s = AllSGPR128s.slice(std::min(static_cast<unsigned>(AllSGPR128s.size()), NumPreloaded));
131 
132  // Skip the last N reserved elements because they should have already been
133  // reserved for VCC etc.
134  for (MCPhysReg Reg : AllSGPR128s) {
135  // Pick the first unallocated one. Make sure we don't clobber the other
136  // reserved input we needed.
137  if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg)) {
138  MRI.replaceRegWith(ScratchRsrcReg, Reg);
139  MFI->setScratchRSrcReg(Reg);
140  return Reg;
141  }
142  }
143 
144  return ScratchRsrcReg;
145 }
146 
147 // Shift down registers reserved for the scratch wave offset and stack pointer
148 // SGPRs.
149 std::pair<unsigned, unsigned>
150 SIFrameLowering::getReservedPrivateSegmentWaveByteOffsetReg(
151  const GCNSubtarget &ST,
152  const SIInstrInfo *TII,
153  const SIRegisterInfo *TRI,
155  MachineFunction &MF) const {
157  unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg();
158 
159  // No replacement necessary.
160  if (ScratchWaveOffsetReg == AMDGPU::NoRegister ||
161  !MRI.isPhysRegUsed(ScratchWaveOffsetReg)) {
162  assert(MFI->getStackPtrOffsetReg() == AMDGPU::SP_REG);
163  return std::make_pair(AMDGPU::NoRegister, AMDGPU::NoRegister);
164  }
165 
166  unsigned SPReg = MFI->getStackPtrOffsetReg();
167  if (ST.hasSGPRInitBug())
168  return std::make_pair(ScratchWaveOffsetReg, SPReg);
169 
170  unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
171 
172  ArrayRef<MCPhysReg> AllSGPRs = getAllSGPRs(ST, MF);
173  if (NumPreloaded > AllSGPRs.size())
174  return std::make_pair(ScratchWaveOffsetReg, SPReg);
175 
176  AllSGPRs = AllSGPRs.slice(NumPreloaded);
177 
178  // We need to drop register from the end of the list that we cannot use
179  // for the scratch wave offset.
180  // + 2 s102 and s103 do not exist on VI.
181  // + 2 for vcc
182  // + 2 for xnack_mask
183  // + 2 for flat_scratch
184  // + 4 for registers reserved for scratch resource register
185  // + 1 for register reserved for scratch wave offset. (By exluding this
186  // register from the list to consider, it means that when this
187  // register is being used for the scratch wave offset and there
188  // are no other free SGPRs, then the value will stay in this register.
189  // + 1 if stack pointer is used.
190  // ----
191  // 13 (+1)
192  unsigned ReservedRegCount = 13;
193 
194  if (AllSGPRs.size() < ReservedRegCount)
195  return std::make_pair(ScratchWaveOffsetReg, SPReg);
196 
197  bool HandledScratchWaveOffsetReg =
198  ScratchWaveOffsetReg != TRI->reservedPrivateSegmentWaveByteOffsetReg(MF);
199 
200  for (MCPhysReg Reg : AllSGPRs.drop_back(ReservedRegCount)) {
201  // Pick the first unallocated SGPR. Be careful not to pick an alias of the
202  // scratch descriptor, since we haven’t added its uses yet.
203  if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg)) {
204  if (!HandledScratchWaveOffsetReg) {
205  HandledScratchWaveOffsetReg = true;
206 
207  MRI.replaceRegWith(ScratchWaveOffsetReg, Reg);
209  ScratchWaveOffsetReg = Reg;
210  break;
211  }
212  }
213  }
214 
215  return std::make_pair(ScratchWaveOffsetReg, SPReg);
216 }
217 
219  MachineBasicBlock &MBB) const {
220  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
221 
223 
224  // If we only have SGPR spills, we won't actually be using scratch memory
225  // since these spill to VGPRs.
226  //
227  // FIXME: We should be cleaning up these unused SGPR spill frame indices
228  // somewhere.
229 
230  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
231  const SIInstrInfo *TII = ST.getInstrInfo();
232  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
234  const Function &F = MF.getFunction();
235 
236  // We need to do the replacement of the private segment buffer and wave offset
237  // register even if there are no stack objects. There could be stores to undef
238  // or a constant without an associated object.
239 
240  // FIXME: We still have implicit uses on SGPR spill instructions in case they
241  // need to spill to vector memory. It's likely that will not happen, but at
242  // this point it appears we need the setup. This part of the prolog should be
243  // emitted after frame indices are eliminated.
244 
245  if (MFI->hasFlatScratchInit())
246  emitFlatScratchInit(ST, MF, MBB);
247 
248  unsigned SPReg = MFI->getStackPtrOffsetReg();
249  if (SPReg != AMDGPU::SP_REG) {
250  assert(MRI.isReserved(SPReg) && "SPReg used but not reserved");
251 
252  DebugLoc DL;
253  const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
254  int64_t StackSize = FrameInfo.getStackSize();
255 
256  if (StackSize == 0) {
257  BuildMI(MBB, MBB.begin(), DL, TII->get(AMDGPU::COPY), SPReg)
258  .addReg(MFI->getScratchWaveOffsetReg());
259  } else {
260  BuildMI(MBB, MBB.begin(), DL, TII->get(AMDGPU::S_ADD_U32), SPReg)
261  .addReg(MFI->getScratchWaveOffsetReg())
262  .addImm(StackSize * ST.getWavefrontSize());
263  }
264  }
265 
266  unsigned ScratchRsrcReg
267  = getReservedPrivateSegmentBufferReg(ST, TII, TRI, MFI, MF);
268 
269  unsigned ScratchWaveOffsetReg;
270  std::tie(ScratchWaveOffsetReg, SPReg)
271  = getReservedPrivateSegmentWaveByteOffsetReg(ST, TII, TRI, MFI, MF);
272 
273  // It's possible to have uses of only ScratchWaveOffsetReg without
274  // ScratchRsrcReg if it's only used for the initialization of flat_scratch,
275  // but the inverse is not true.
276  if (ScratchWaveOffsetReg == AMDGPU::NoRegister) {
277  assert(ScratchRsrcReg == AMDGPU::NoRegister);
278  return;
279  }
280 
281  // We need to insert initialization of the scratch resource descriptor.
282  unsigned PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
284 
285  unsigned PreloadedPrivateBufferReg = AMDGPU::NoRegister;
286  if (ST.isAmdHsaOrMesa(F)) {
287  PreloadedPrivateBufferReg = MFI->getPreloadedReg(
289  }
290 
291  bool OffsetRegUsed = MRI.isPhysRegUsed(ScratchWaveOffsetReg);
292  bool ResourceRegUsed = ScratchRsrcReg != AMDGPU::NoRegister &&
293  MRI.isPhysRegUsed(ScratchRsrcReg);
294 
295  // We added live-ins during argument lowering, but since they were not used
296  // they were deleted. We're adding the uses now, so add them back.
297  if (OffsetRegUsed) {
298  assert(PreloadedScratchWaveOffsetReg != AMDGPU::NoRegister &&
299  "scratch wave offset input is required");
300  MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
301  MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
302  }
303 
304  if (ResourceRegUsed && PreloadedPrivateBufferReg != AMDGPU::NoRegister) {
305  assert(ST.isAmdHsaOrMesa(F) || ST.isMesaGfxShader(F));
306  MRI.addLiveIn(PreloadedPrivateBufferReg);
307  MBB.addLiveIn(PreloadedPrivateBufferReg);
308  }
309 
310  // Make the register selected live throughout the function.
311  for (MachineBasicBlock &OtherBB : MF) {
312  if (&OtherBB == &MBB)
313  continue;
314 
315  if (OffsetRegUsed)
316  OtherBB.addLiveIn(ScratchWaveOffsetReg);
317 
318  if (ResourceRegUsed)
319  OtherBB.addLiveIn(ScratchRsrcReg);
320  }
321 
322  DebugLoc DL;
324 
325  // If we reserved the original input registers, we don't need to copy to the
326  // reserved registers.
327 
328  bool CopyBuffer = ResourceRegUsed &&
329  PreloadedPrivateBufferReg != AMDGPU::NoRegister &&
330  ST.isAmdHsaOrMesa(F) &&
331  ScratchRsrcReg != PreloadedPrivateBufferReg;
332 
333  // This needs to be careful of the copying order to avoid overwriting one of
334  // the input registers before it's been copied to it's final
335  // destination. Usually the offset should be copied first.
336  bool CopyBufferFirst = TRI->isSubRegisterEq(PreloadedPrivateBufferReg,
337  ScratchWaveOffsetReg);
338  if (CopyBuffer && CopyBufferFirst) {
339  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
340  .addReg(PreloadedPrivateBufferReg, RegState::Kill);
341  }
342 
343  if (OffsetRegUsed &&
344  PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg) {
345  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg)
346  .addReg(PreloadedScratchWaveOffsetReg,
347  MRI.isPhysRegUsed(ScratchWaveOffsetReg) ? 0 : RegState::Kill);
348  }
349 
350  if (CopyBuffer && !CopyBufferFirst) {
351  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
352  .addReg(PreloadedPrivateBufferReg, RegState::Kill);
353  }
354 
355  if (ResourceRegUsed)
356  emitEntryFunctionScratchSetup(ST, MF, MBB, MFI, I,
357  PreloadedPrivateBufferReg, ScratchRsrcReg);
358 }
359 
360 // Emit scratch setup code for AMDPAL or Mesa, assuming ResourceRegUsed is set.
361 void SIFrameLowering::emitEntryFunctionScratchSetup(const GCNSubtarget &ST,
363  MachineBasicBlock::iterator I, unsigned PreloadedPrivateBufferReg,
364  unsigned ScratchRsrcReg) const {
365 
366  const SIInstrInfo *TII = ST.getInstrInfo();
367  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
368  const Function &Fn = MF.getFunction();
369  DebugLoc DL;
370 
371  if (ST.isAmdPalOS()) {
372  // The pointer to the GIT is formed from the offset passed in and either
373  // the amdgpu-git-ptr-high function attribute or the top part of the PC
374  unsigned RsrcLo = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
375  unsigned RsrcHi = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
376  unsigned Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
377 
378  const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
379 
380  if (MFI->getGITPtrHigh() != 0xffffffff) {
381  BuildMI(MBB, I, DL, SMovB32, RsrcHi)
382  .addImm(MFI->getGITPtrHigh())
383  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
384  } else {
385  const MCInstrDesc &GetPC64 = TII->get(AMDGPU::S_GETPC_B64);
386  BuildMI(MBB, I, DL, GetPC64, Rsrc01);
387  }
388  auto GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
389  if (ST.hasMergedShaders()) {
390  switch (MF.getFunction().getCallingConv()) {
393  // Low GIT address is passed in s8 rather than s0 for an LS+HS or
394  // ES+GS merged shader on gfx9+.
395  GitPtrLo = AMDGPU::SGPR8;
396  break;
397  default:
398  break;
399  }
400  }
401  MF.getRegInfo().addLiveIn(GitPtrLo);
402  MF.front().addLiveIn(GitPtrLo);
403  BuildMI(MBB, I, DL, SMovB32, RsrcLo)
404  .addReg(GitPtrLo)
405  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
406 
407  // We now have the GIT ptr - now get the scratch descriptor from the entry
408  // at offset 0 (or offset 16 for a compute shader).
409  PointerType *PtrTy =
412  MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
413  const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
414  auto MMO = MF.getMachineMemOperand(PtrInfo,
418  16, 4);
419  unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
420  const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
421  unsigned EncodedOffset = AMDGPU::getSMRDEncodedOffset(Subtarget, Offset);
422  BuildMI(MBB, I, DL, LoadDwordX4, ScratchRsrcReg)
423  .addReg(Rsrc01)
424  .addImm(EncodedOffset) // offset
425  .addImm(0) // glc
426  .addReg(ScratchRsrcReg, RegState::ImplicitDefine)
427  .addMemOperand(MMO);
428  return;
429  }
430  if (ST.isMesaGfxShader(Fn)
431  || (PreloadedPrivateBufferReg == AMDGPU::NoRegister)) {
432  assert(!ST.isAmdHsaOrMesa(Fn));
433  const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
434 
435  unsigned Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
436  unsigned Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
437 
438  // Use relocations to get the pointer, and setup the other bits manually.
439  uint64_t Rsrc23 = TII->getScratchRsrcWords23();
440 
441  if (MFI->hasImplicitBufferPtr()) {
442  unsigned Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
443 
445  const MCInstrDesc &Mov64 = TII->get(AMDGPU::S_MOV_B64);
446 
447  BuildMI(MBB, I, DL, Mov64, Rsrc01)
449  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
450  } else {
451  const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
452 
453  PointerType *PtrTy =
456  MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
457  auto MMO = MF.getMachineMemOperand(PtrInfo,
461  8, 4);
462  BuildMI(MBB, I, DL, LoadDwordX2, Rsrc01)
464  .addImm(0) // offset
465  .addImm(0) // glc
466  .addMemOperand(MMO)
467  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
468  }
469  } else {
470  unsigned Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
471  unsigned Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
472 
473  BuildMI(MBB, I, DL, SMovB32, Rsrc0)
474  .addExternalSymbol("SCRATCH_RSRC_DWORD0")
475  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
476 
477  BuildMI(MBB, I, DL, SMovB32, Rsrc1)
478  .addExternalSymbol("SCRATCH_RSRC_DWORD1")
479  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
480 
481  }
482 
483  BuildMI(MBB, I, DL, SMovB32, Rsrc2)
484  .addImm(Rsrc23 & 0xffffffff)
485  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
486 
487  BuildMI(MBB, I, DL, SMovB32, Rsrc3)
488  .addImm(Rsrc23 >> 32)
489  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
490  }
491 }
492 
493 // Find a scratch register that we can use at the start of the prologue to
494 // re-align the stack pointer. We avoid using callee-save registers since they
495 // may appear to be free when this is called from canUseAsPrologue (during
496 // shrink wrapping), but then no longer be free when this is called from
497 // emitPrologue.
498 //
499 // FIXME: This is a bit conservative, since in the above case we could use one
500 // of the callee-save registers as a scratch temp to re-align the stack pointer,
501 // but we would then have to make sure that we were in fact saving at least one
502 // callee-save register in the prologue, which is additional complexity that
503 // doesn't seem worth the benefit.
505  MachineFunction *MF = MBB.getParent();
506 
507  const GCNSubtarget &Subtarget = MF->getSubtarget<GCNSubtarget>();
508  const SIRegisterInfo &TRI = *Subtarget.getRegisterInfo();
509  LivePhysRegs LiveRegs(TRI);
510  LiveRegs.addLiveIns(MBB);
511 
512  // Mark callee saved registers as used so we will not choose them.
513  const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF);
514  for (unsigned i = 0; CSRegs[i]; ++i)
515  LiveRegs.addReg(CSRegs[i]);
516 
518 
519  for (unsigned Reg : AMDGPU::SReg_32_XM0RegClass) {
520  if (LiveRegs.available(MRI, Reg))
521  return Reg;
522  }
523 
524  return AMDGPU::NoRegister;
525 }
526 
528  MachineBasicBlock &MBB) const {
530  if (FuncInfo->isEntryFunction()) {
531  emitEntryFunctionPrologue(MF, MBB);
532  return;
533  }
534 
535  const MachineFrameInfo &MFI = MF.getFrameInfo();
536  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
537  const SIInstrInfo *TII = ST.getInstrInfo();
538  const SIRegisterInfo &TRI = TII->getRegisterInfo();
539 
540  unsigned StackPtrReg = FuncInfo->getStackPtrOffsetReg();
541  unsigned FramePtrReg = FuncInfo->getFrameOffsetReg();
542 
543  MachineBasicBlock::iterator MBBI = MBB.begin();
544  DebugLoc DL;
545 
546  // XXX - Is this the right predicate?
547 
548  bool NeedFP = hasFP(MF);
549  uint32_t NumBytes = MFI.getStackSize();
550  uint32_t RoundedSize = NumBytes;
551  const bool NeedsRealignment = TRI.needsStackRealignment(MF);
552 
553  if (NeedsRealignment) {
554  assert(NeedFP);
555  const unsigned Alignment = MFI.getMaxAlignment();
556 
557  RoundedSize += Alignment;
558 
559  unsigned ScratchSPReg = findScratchNonCalleeSaveRegister(MBB);
560  assert(ScratchSPReg != AMDGPU::NoRegister);
561 
562  // s_add_u32 tmp_reg, s32, NumBytes
563  // s_and_b32 s32, tmp_reg, 0b111...0000
564  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_U32), ScratchSPReg)
565  .addReg(StackPtrReg)
566  .addImm((Alignment - 1) * ST.getWavefrontSize())
567  .setMIFlag(MachineInstr::FrameSetup);
568  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_AND_B32), FramePtrReg)
569  .addReg(ScratchSPReg, RegState::Kill)
570  .addImm(-Alignment * ST.getWavefrontSize())
571  .setMIFlag(MachineInstr::FrameSetup);
572  FuncInfo->setIsStackRealigned(true);
573  } else if (NeedFP) {
574  // If we need a base pointer, set it up here. It's whatever the value of
575  // the stack pointer is at this point. Any variable size objects will be
576  // allocated after this, so we can still use the base pointer to reference
577  // locals.
578  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
579  .addReg(StackPtrReg)
581  }
582 
583  if (RoundedSize != 0 && hasSP(MF)) {
584  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_U32), StackPtrReg)
585  .addReg(StackPtrReg)
586  .addImm(RoundedSize * ST.getWavefrontSize())
587  .setMIFlag(MachineInstr::FrameSetup);
588  }
589 
591  : FuncInfo->getSGPRSpillVGPRs()) {
592  if (!Reg.FI.hasValue())
593  continue;
594  TII->storeRegToStackSlot(MBB, MBBI, Reg.VGPR, true,
595  Reg.FI.getValue(), &AMDGPU::VGPR_32RegClass,
596  &TII->getRegisterInfo());
597  }
598 }
599 
601  MachineBasicBlock &MBB) const {
602  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
603  if (FuncInfo->isEntryFunction())
604  return;
605 
606  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
607  const SIInstrInfo *TII = ST.getInstrInfo();
609 
611  : FuncInfo->getSGPRSpillVGPRs()) {
612  if (!Reg.FI.hasValue())
613  continue;
614  TII->loadRegFromStackSlot(MBB, MBBI, Reg.VGPR,
615  Reg.FI.getValue(), &AMDGPU::VGPR_32RegClass,
616  &TII->getRegisterInfo());
617  }
618 
619  unsigned StackPtrReg = FuncInfo->getStackPtrOffsetReg();
620  if (StackPtrReg == AMDGPU::NoRegister)
621  return;
622 
623  const MachineFrameInfo &MFI = MF.getFrameInfo();
624  uint32_t NumBytes = MFI.getStackSize();
625 
626  DebugLoc DL;
627 
628  // FIXME: Clarify distinction between no set SP and SP. For callee functions,
629  // it's really whether we need SP to be accurate or not.
630 
631  if (NumBytes != 0 && hasSP(MF)) {
632  uint32_t RoundedSize = FuncInfo->isStackRealigned() ?
633  NumBytes + MFI.getMaxAlignment() : NumBytes;
634 
635  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_SUB_U32), StackPtrReg)
636  .addReg(StackPtrReg)
637  .addImm(RoundedSize * ST.getWavefrontSize());
638  }
639 }
640 
641 static bool allStackObjectsAreDead(const MachineFrameInfo &MFI) {
642  for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
643  I != E; ++I) {
644  if (!MFI.isDeadObjectIndex(I))
645  return false;
646  }
647 
648  return true;
649 }
650 
652  unsigned &FrameReg) const {
653  const SIRegisterInfo *RI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
654 
655  FrameReg = RI->getFrameRegister(MF);
656  return MF.getFrameInfo().getObjectOffset(FI);
657 }
658 
660  MachineFunction &MF,
661  RegScavenger *RS) const {
662  MachineFrameInfo &MFI = MF.getFrameInfo();
663 
664  if (!MFI.hasStackObjects())
665  return;
666 
667  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
668  const SIInstrInfo *TII = ST.getInstrInfo();
669  const SIRegisterInfo &TRI = TII->getRegisterInfo();
671  bool AllSGPRSpilledToVGPRs = false;
672 
673  if (TRI.spillSGPRToVGPR() && FuncInfo->hasSpilledSGPRs()) {
674  AllSGPRSpilledToVGPRs = true;
675 
676  // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
677  // are spilled to VGPRs, in which case we can eliminate the stack usage.
678  //
679  // XXX - This operates under the assumption that only other SGPR spills are
680  // users of the frame index. I'm not 100% sure this is correct. The
681  // StackColoring pass has a comment saying a future improvement would be to
682  // merging of allocas with spill slots, but for now according to
683  // MachineFrameInfo isSpillSlot can't alias any other object.
684  for (MachineBasicBlock &MBB : MF) {
686  for (auto I = MBB.begin(), E = MBB.end(); I != E; I = Next) {
687  MachineInstr &MI = *I;
688  Next = std::next(I);
689 
690  if (TII->isSGPRSpill(MI)) {
691  int FI = TII->getNamedOperand(MI, AMDGPU::OpName::addr)->getIndex();
693  if (FuncInfo->allocateSGPRSpillToVGPR(MF, FI)) {
694  bool Spilled = TRI.eliminateSGPRToVGPRSpillFrameIndex(MI, FI, RS);
695  (void)Spilled;
696  assert(Spilled && "failed to spill SGPR to VGPR when allocated");
697  } else
698  AllSGPRSpilledToVGPRs = false;
699  }
700  }
701  }
702  }
703 
704  FuncInfo->removeSGPRToVGPRFrameIndices(MFI);
705 
706  // FIXME: The other checks should be redundant with allStackObjectsAreDead,
707  // but currently hasNonSpillStackObjects is set only from source
708  // allocas. Stack temps produced from legalization are not counted currently.
709  if (FuncInfo->hasNonSpillStackObjects() || FuncInfo->hasSpilledVGPRs() ||
710  !AllSGPRSpilledToVGPRs || !allStackObjectsAreDead(MFI)) {
711  assert(RS && "RegScavenger required if spilling");
712 
713  // We force this to be at offset 0 so no user object ever has 0 as an
714  // address, so we may use 0 as an invalid pointer value. This is because
715  // LLVM assumes 0 is an invalid pointer in address space 0. Because alloca
716  // is required to be address space 0, we are forced to accept this for
717  // now. Ideally we could have the stack in another address space with 0 as a
718  // valid pointer, and -1 as the null value.
719  //
720  // This will also waste additional space when user stack objects require > 4
721  // byte alignment.
722  //
723  // The main cost here is losing the offset for addressing modes. However
724  // this also ensures we shouldn't need a register for the offset when
725  // emergency scavenging.
726  int ScavengeFI = MFI.CreateFixedObject(
727  TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0, false);
728  RS->addScavengingFrameIndex(ScavengeFI);
729  }
730 }
731 
733  RegScavenger *RS) const {
736 
737  // The SP is specifically managed and we don't want extra spills of it.
738  SavedRegs.reset(MFI->getStackPtrOffsetReg());
739 }
740 
742  MachineFunction &MF,
743  MachineBasicBlock &MBB,
744  MachineBasicBlock::iterator I) const {
745  int64_t Amount = I->getOperand(0).getImm();
746  if (Amount == 0)
747  return MBB.erase(I);
748 
749  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
750  const SIInstrInfo *TII = ST.getInstrInfo();
751  const DebugLoc &DL = I->getDebugLoc();
752  unsigned Opc = I->getOpcode();
753  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
754  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
755 
757  if (!TFI->hasReservedCallFrame(MF)) {
758  unsigned Align = getStackAlignment();
759 
760  Amount = alignTo(Amount, Align);
761  assert(isUInt<32>(Amount) && "exceeded stack address space size");
763  unsigned SPReg = MFI->getStackPtrOffsetReg();
764 
765  unsigned Op = IsDestroy ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
766  BuildMI(MBB, I, DL, TII->get(Op), SPReg)
767  .addReg(SPReg)
768  .addImm(Amount * ST.getWavefrontSize());
769  } else if (CalleePopAmount != 0) {
770  llvm_unreachable("is this used?");
771  }
772 
773  return MBB.erase(I);
774 }
775 
777  // All stack operations are relative to the frame offset SGPR.
778  // TODO: Still want to eliminate sometimes.
779  const MachineFrameInfo &MFI = MF.getFrameInfo();
780 
781  // XXX - Is this only called after frame is finalized? Should be able to check
782  // frame size.
783  return MFI.hasStackObjects() && !allStackObjectsAreDead(MFI);
784 }
785 
787  const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
788  // All stack operations are relative to the frame offset SGPR.
789  const MachineFrameInfo &MFI = MF.getFrameInfo();
790  return MFI.hasCalls() || MFI.hasVarSizedObjects() || TRI->needsStackRealignment(MF);
791 }
constexpr bool isUInt< 32 >(uint64_t x)
Definition: MathExtras.h:348
int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const override
getFrameIndexReference - This method should return the base register and offset used to reference a f...
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
Interface definition for SIRegisterInfo.
unsigned reservedPrivateSegmentBufferReg(const MachineFunction &MF) const
Return the end register initially reserved for the scratch buffer in case spilling is needed...
AMDGPU specific subclass of TargetSubtarget.
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:224
static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock &MBB)
bool isAllocatable(unsigned PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn&#39;t been...
This class represents lattice values for constants.
Definition: AllocatorList.h:23
bool hasStackObjects() const
Return true if there are any stack objects in this function.
void addLiveIn(unsigned Reg, unsigned vreg=0)
addLiveIn - Add the specified register as a live-in.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS=nullptr) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
void removeSGPRToVGPRFrameIndices(MachineFrameInfo &MFI)
unsigned Reg
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space...
Definition: Type.cpp:629
void emitEntryFunctionPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const
const SIInstrInfo * getInstrInfo() const override
bool hasMergedShaders() const
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
F(f)
uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:684
void setIsStackRealigned(bool Realigned=true)
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:176
const SIRegisterInfo & getRegisterInfo() const
Definition: SIInstrInfo.h:165
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
Calling convention used for Mesa/AMDPAL compute shaders.
Definition: CallingConv.h:197
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
bool isMesaGfxShader(const Function &F) const
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
const HexagonInstrInfo * TII
uint64_t getScratchRsrcWords23() const
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
static bool allStackObjectsAreDead(const MachineFrameInfo &MFI)
The memory access is dereferenceable (i.e., doesn&#39;t trap).
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
int getObjectIndexBegin() const
Return the minimum frame object index.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI)
Reserve a slice of a VGPR to support spilling for FrameIndex FI.
Class to represent pointers.
Definition: DerivedTypes.h:498
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1165
int64_t getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset)
bool isCompute(CallingConv::ID cc)
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
This file declares the machine register scavenger class.
unsigned const MachineRegisterInfo * MRI
unsigned getFrameRegister(const MachineFunction &MF) const override
unsigned reservedPrivateSegmentWaveByteOffsetReg(const MachineFunction &MF) const
Return the end register initially reserved for the scratch wave offset in case spilling is needed...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
static ArrayRef< MCPhysReg > getAllSGPR128(const GCNSubtarget &ST, const MachineFunction &MF)
Calling convention used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
Definition: CallingConv.h:207
static ArrayRef< MCPhysReg > getAllSGPRs(const GCNSubtarget &ST, const MachineFunction &MF)
BitVector & reset()
Definition: BitVector.h:438
unsigned getMaxAlignment() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Address space for constant memory (VTX2).
Definition: AMDGPU.h:254
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:192
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1424
ArrayRef< SGPRSpillVGPRCSR > getSGPRSpillVGPRs() const
void setScratchWaveOffsetReg(unsigned Reg)
const MachineBasicBlock & front() const
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
This class contains a discriminated union of information about pointers in memory operands...
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
virtual bool hasReservedCallFrame(const MachineFunction &MF) const
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required, we reserve argument space for call sites in the function immediately on entry to the current function.
unsigned getWavefrontSize() const
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:212
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
unsigned getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses...
Information about stack frame layout on the target.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
Calling convention used for Mesa/AMDPAL geometry shaders.
Definition: CallingConv.h:191
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
bool hasSGPRInitBug() const
ArrayRef< T > drop_back(size_t N=1) const
Drop the last N elements of the array.
Definition: ArrayRef.h:193
const Function & getFunction() const
Return the LLVM function that this machine code represents.
bool isPhysRegUsed(unsigned PhysReg) const
Return true if the specified register is modified or read in this function.
bool eliminateSGPRToVGPRSpillFrameIndex(MachineBasicBlock::iterator MI, int FI, RegScavenger *RS) const
Special case of eliminateFrameIndex.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array...
Definition: ArrayRef.h:178
void replaceRegWith(unsigned FromReg, unsigned ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
bool flatScratchIsPointer() const
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Interface definition for SIInstrInfo.
unsigned getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:48
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned char TargetFlags=0) const
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned getImplicitBufferPtrUserSGPR() const
virtual const TargetFrameLowering * getFrameLowering() const
The memory access always returns the same value (or traps).
bool isAmdHsaOrMesa(const Function &F) const
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool spillSGPRToVGPR() const
bool hasSP(const MachineFunction &MF) const
uint8_t getStackID(int ObjectIdx) const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
Definition: LivePhysRegs.h:79
IRTranslator LLVM IR MI
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects...
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register...
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
bool hasCalls() const
Return true if the current function has any function calls.
const SIRegisterInfo * getRegisterInfo() const override