LLVM  6.0.0svn
SIFrameLowering.cpp
Go to the documentation of this file.
1 //===----------------------- SIFrameLowering.cpp --------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //==-----------------------------------------------------------------------===//
9 
10 #include "SIFrameLowering.h"
11 #include "AMDGPUSubtarget.h"
12 #include "SIInstrInfo.h"
13 #include "SIMachineFunctionInfo.h"
14 #include "SIRegisterInfo.h"
15 
20 
21 using namespace llvm;
22 
23 
25  const MachineFunction &MF) {
26  return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(),
27  ST.getMaxNumSGPRs(MF) / 4);
28 }
29 
31  const MachineFunction &MF) {
32  return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(),
33  ST.getMaxNumSGPRs(MF));
34 }
35 
36 void SIFrameLowering::emitFlatScratchInit(const SISubtarget &ST,
37  MachineFunction &MF,
38  MachineBasicBlock &MBB) const {
39  const SIInstrInfo *TII = ST.getInstrInfo();
40  const SIRegisterInfo* TRI = &TII->getRegisterInfo();
42 
43  // We don't need this if we only have spills since there is no user facing
44  // scratch.
45 
46  // TODO: If we know we don't have flat instructions earlier, we can omit
47  // this from the input registers.
48  //
49  // TODO: We only need to know if we access scratch space through a flat
50  // pointer. Because we only detect if flat instructions are used at all,
51  // this will be used more often than necessary on VI.
52 
53  // Debug location must be unknown since the first debug location is used to
54  // determine the end of the prologue.
55  DebugLoc DL;
57 
58  unsigned FlatScratchInitReg
60 
62  MRI.addLiveIn(FlatScratchInitReg);
63  MBB.addLiveIn(FlatScratchInitReg);
64 
65  unsigned FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0);
66  unsigned FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1);
67 
68  unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg();
69 
70  // Do a 64-bit pointer add.
71  if (ST.flatScratchIsPointer()) {
72  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), AMDGPU::FLAT_SCR_LO)
73  .addReg(FlatScrInitLo)
74  .addReg(ScratchWaveOffsetReg);
75  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), AMDGPU::FLAT_SCR_HI)
76  .addReg(FlatScrInitHi)
77  .addImm(0);
78 
79  return;
80  }
81 
82  // Copy the size in bytes.
83  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::FLAT_SCR_LO)
84  .addReg(FlatScrInitHi, RegState::Kill);
85 
86  // Add wave offset in bytes to private base offset.
87  // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init.
88  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo)
89  .addReg(FlatScrInitLo)
90  .addReg(ScratchWaveOffsetReg);
91 
92  // Convert offset to 256-byte units.
93  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI)
94  .addReg(FlatScrInitLo, RegState::Kill)
95  .addImm(8);
96 }
97 
98 unsigned SIFrameLowering::getReservedPrivateSegmentBufferReg(
99  const SISubtarget &ST,
100  const SIInstrInfo *TII,
101  const SIRegisterInfo *TRI,
103  MachineFunction &MF) const {
105 
106  // We need to insert initialization of the scratch resource descriptor.
107  unsigned ScratchRsrcReg = MFI->getScratchRSrcReg();
108  if (ScratchRsrcReg == AMDGPU::NoRegister ||
109  !MRI.isPhysRegUsed(ScratchRsrcReg))
110  return AMDGPU::NoRegister;
111 
112  if (ST.hasSGPRInitBug() ||
113  ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
114  return ScratchRsrcReg;
115 
116  // We reserved the last registers for this. Shift it down to the end of those
117  // which were actually used.
118  //
119  // FIXME: It might be safer to use a pseudoregister before replacement.
120 
121  // FIXME: We should be able to eliminate unused input registers. We only
122  // cannot do this for the resources required for scratch access. For now we
123  // skip over user SGPRs and may leave unused holes.
124 
125  // We find the resource first because it has an alignment requirement.
126 
127  unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4;
128  ArrayRef<MCPhysReg> AllSGPR128s = getAllSGPR128(ST, MF);
129  AllSGPR128s = AllSGPR128s.slice(std::min(static_cast<unsigned>(AllSGPR128s.size()), NumPreloaded));
130 
131  // Skip the last N reserved elements because they should have already been
132  // reserved for VCC etc.
133  for (MCPhysReg Reg : AllSGPR128s) {
134  // Pick the first unallocated one. Make sure we don't clobber the other
135  // reserved input we needed.
136  if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg)) {
137  MRI.replaceRegWith(ScratchRsrcReg, Reg);
138  MFI->setScratchRSrcReg(Reg);
139  return Reg;
140  }
141  }
142 
143  return ScratchRsrcReg;
144 }
145 
146 // Shift down registers reserved for the scratch wave offset and stack pointer
147 // SGPRs.
148 std::pair<unsigned, unsigned>
149 SIFrameLowering::getReservedPrivateSegmentWaveByteOffsetReg(
150  const SISubtarget &ST,
151  const SIInstrInfo *TII,
152  const SIRegisterInfo *TRI,
154  MachineFunction &MF) const {
156  unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg();
157 
158  // No replacement necessary.
159  if (ScratchWaveOffsetReg == AMDGPU::NoRegister ||
160  !MRI.isPhysRegUsed(ScratchWaveOffsetReg)) {
161  assert(MFI->getStackPtrOffsetReg() == AMDGPU::SP_REG);
162  return std::make_pair(AMDGPU::NoRegister, AMDGPU::NoRegister);
163  }
164 
165  unsigned SPReg = MFI->getStackPtrOffsetReg();
166  if (ST.hasSGPRInitBug())
167  return std::make_pair(ScratchWaveOffsetReg, SPReg);
168 
169  unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
170 
171  ArrayRef<MCPhysReg> AllSGPRs = getAllSGPRs(ST, MF);
172  if (NumPreloaded > AllSGPRs.size())
173  return std::make_pair(ScratchWaveOffsetReg, SPReg);
174 
175  AllSGPRs = AllSGPRs.slice(NumPreloaded);
176 
177  // We need to drop register from the end of the list that we cannot use
178  // for the scratch wave offset.
179  // + 2 s102 and s103 do not exist on VI.
180  // + 2 for vcc
181  // + 2 for xnack_mask
182  // + 2 for flat_scratch
183  // + 4 for registers reserved for scratch resource register
184  // + 1 for register reserved for scratch wave offset. (By exluding this
185  // register from the list to consider, it means that when this
186  // register is being used for the scratch wave offset and there
187  // are no other free SGPRs, then the value will stay in this register.
188  // + 1 if stack pointer is used.
189  // ----
190  // 13 (+1)
191  unsigned ReservedRegCount = 13;
192 
193  if (AllSGPRs.size() < ReservedRegCount)
194  return std::make_pair(ScratchWaveOffsetReg, SPReg);
195 
196  bool HandledScratchWaveOffsetReg =
197  ScratchWaveOffsetReg != TRI->reservedPrivateSegmentWaveByteOffsetReg(MF);
198 
199  for (MCPhysReg Reg : AllSGPRs.drop_back(ReservedRegCount)) {
200  // Pick the first unallocated SGPR. Be careful not to pick an alias of the
201  // scratch descriptor, since we haven’t added its uses yet.
202  if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg)) {
203  if (!HandledScratchWaveOffsetReg) {
204  HandledScratchWaveOffsetReg = true;
205 
206  MRI.replaceRegWith(ScratchWaveOffsetReg, Reg);
208  ScratchWaveOffsetReg = Reg;
209  break;
210  }
211  }
212  }
213 
214  return std::make_pair(ScratchWaveOffsetReg, SPReg);
215 }
216 
218  MachineBasicBlock &MBB) const {
219  // Emit debugger prologue if "amdgpu-debugger-emit-prologue" attribute was
220  // specified.
221  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
222  if (ST.debuggerEmitPrologue())
223  emitDebuggerPrologue(MF, MBB);
224 
225  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
226 
228 
229  // If we only have SGPR spills, we won't actually be using scratch memory
230  // since these spill to VGPRs.
231  //
232  // FIXME: We should be cleaning up these unused SGPR spill frame indices
233  // somewhere.
234 
235  const SIInstrInfo *TII = ST.getInstrInfo();
236  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
238 
239  // We need to do the replacement of the private segment buffer and wave offset
240  // register even if there are no stack objects. There could be stores to undef
241  // or a constant without an associated object.
242 
243  // FIXME: We still have implicit uses on SGPR spill instructions in case they
244  // need to spill to vector memory. It's likely that will not happen, but at
245  // this point it appears we need the setup. This part of the prolog should be
246  // emitted after frame indices are eliminated.
247 
248  if (MFI->hasFlatScratchInit())
249  emitFlatScratchInit(ST, MF, MBB);
250 
251  unsigned SPReg = MFI->getStackPtrOffsetReg();
252  if (SPReg != AMDGPU::SP_REG) {
253  assert(MRI.isReserved(SPReg) && "SPReg used but not reserved");
254 
255  DebugLoc DL;
256  const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
257  int64_t StackSize = FrameInfo.getStackSize();
258 
259  if (StackSize == 0) {
260  BuildMI(MBB, MBB.begin(), DL, TII->get(AMDGPU::COPY), SPReg)
261  .addReg(MFI->getScratchWaveOffsetReg());
262  } else {
263  BuildMI(MBB, MBB.begin(), DL, TII->get(AMDGPU::S_ADD_U32), SPReg)
264  .addReg(MFI->getScratchWaveOffsetReg())
265  .addImm(StackSize * ST.getWavefrontSize());
266  }
267  }
268 
269  unsigned ScratchRsrcReg
270  = getReservedPrivateSegmentBufferReg(ST, TII, TRI, MFI, MF);
271 
272  unsigned ScratchWaveOffsetReg;
273  std::tie(ScratchWaveOffsetReg, SPReg)
274  = getReservedPrivateSegmentWaveByteOffsetReg(ST, TII, TRI, MFI, MF);
275 
276  // It's possible to have uses of only ScratchWaveOffsetReg without
277  // ScratchRsrcReg if it's only used for the initialization of flat_scratch,
278  // but the inverse is not true.
279  if (ScratchWaveOffsetReg == AMDGPU::NoRegister) {
280  assert(ScratchRsrcReg == AMDGPU::NoRegister);
281  return;
282  }
283 
284  // We need to insert initialization of the scratch resource descriptor.
285  unsigned PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
287 
288  unsigned PreloadedPrivateBufferReg = AMDGPU::NoRegister;
289  if (ST.isAmdCodeObjectV2(MF)) {
290  PreloadedPrivateBufferReg = MFI->getPreloadedReg(
292  }
293 
294  bool OffsetRegUsed = MRI.isPhysRegUsed(ScratchWaveOffsetReg);
295  bool ResourceRegUsed = ScratchRsrcReg != AMDGPU::NoRegister &&
296  MRI.isPhysRegUsed(ScratchRsrcReg);
297 
298  // We added live-ins during argument lowering, but since they were not used
299  // they were deleted. We're adding the uses now, so add them back.
300  if (OffsetRegUsed) {
301  assert(PreloadedScratchWaveOffsetReg != AMDGPU::NoRegister &&
302  "scratch wave offset input is required");
303  MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
304  MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
305  }
306 
307  if (ResourceRegUsed && PreloadedPrivateBufferReg != AMDGPU::NoRegister) {
308  assert(ST.isAmdCodeObjectV2(MF) || ST.isMesaGfxShader(MF));
309  MRI.addLiveIn(PreloadedPrivateBufferReg);
310  MBB.addLiveIn(PreloadedPrivateBufferReg);
311  }
312 
313  // Make the register selected live throughout the function.
314  for (MachineBasicBlock &OtherBB : MF) {
315  if (&OtherBB == &MBB)
316  continue;
317 
318  if (OffsetRegUsed)
319  OtherBB.addLiveIn(ScratchWaveOffsetReg);
320 
321  if (ResourceRegUsed)
322  OtherBB.addLiveIn(ScratchRsrcReg);
323  }
324 
325  DebugLoc DL;
327 
328  // If we reserved the original input registers, we don't need to copy to the
329  // reserved registers.
330 
331  bool CopyBuffer = ResourceRegUsed &&
332  PreloadedPrivateBufferReg != AMDGPU::NoRegister &&
333  ST.isAmdCodeObjectV2(MF) &&
334  ScratchRsrcReg != PreloadedPrivateBufferReg;
335 
336  // This needs to be careful of the copying order to avoid overwriting one of
337  // the input registers before it's been copied to it's final
338  // destination. Usually the offset should be copied first.
339  bool CopyBufferFirst = TRI->isSubRegisterEq(PreloadedPrivateBufferReg,
340  ScratchWaveOffsetReg);
341  if (CopyBuffer && CopyBufferFirst) {
342  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
343  .addReg(PreloadedPrivateBufferReg, RegState::Kill);
344  }
345 
346  if (OffsetRegUsed &&
347  PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg) {
348  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg)
349  .addReg(PreloadedScratchWaveOffsetReg,
350  MRI.isPhysRegUsed(ScratchWaveOffsetReg) ? 0 : RegState::Kill);
351  }
352 
353  if (CopyBuffer && !CopyBufferFirst) {
354  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
355  .addReg(PreloadedPrivateBufferReg, RegState::Kill);
356  }
357 
358  if (ResourceRegUsed)
359  emitEntryFunctionScratchSetup(ST, MF, MBB, MFI, I,
360  PreloadedPrivateBufferReg, ScratchRsrcReg);
361 }
362 
363 // Emit scratch setup code for AMDPAL or Mesa, assuming ResourceRegUsed is set.
364 void SIFrameLowering::emitEntryFunctionScratchSetup(const SISubtarget &ST,
366  MachineBasicBlock::iterator I, unsigned PreloadedPrivateBufferReg,
367  unsigned ScratchRsrcReg) const {
368 
369  const SIInstrInfo *TII = ST.getInstrInfo();
370  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
371  DebugLoc DL;
372 
373  if (ST.isAmdPalOS()) {
374  // The pointer to the GIT is formed from the offset passed in and either
375  // the amdgpu-git-ptr-high function attribute or the top part of the PC
376  unsigned RsrcLo = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
377  unsigned RsrcHi = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
378  unsigned Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
379 
380  const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
381 
382  if (MFI->getGITPtrHigh() != 0xffffffff) {
383  BuildMI(MBB, I, DL, SMovB32, RsrcHi)
384  .addImm(MFI->getGITPtrHigh())
385  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
386  } else {
387  const MCInstrDesc &GetPC64 = TII->get(AMDGPU::S_GETPC_B64);
388  BuildMI(MBB, I, DL, GetPC64, Rsrc01);
389  }
390  BuildMI(MBB, I, DL, SMovB32, RsrcLo)
391  .addReg(AMDGPU::SGPR0) // Low address passed in
392  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
393 
394  // We now have the GIT ptr - now get the scratch descriptor from the entry
395  // at offset 0.
396  PointerType *PtrTy =
399  MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
400  const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
401  auto MMO = MF.getMachineMemOperand(PtrInfo,
405  0, 0);
406  BuildMI(MBB, I, DL, LoadDwordX4, ScratchRsrcReg)
407  .addReg(Rsrc01)
408  .addImm(0) // offset
409  .addImm(0) // glc
410  .addReg(ScratchRsrcReg, RegState::ImplicitDefine)
411  .addMemOperand(MMO);
412  return;
413  }
414  if (ST.isMesaGfxShader(MF)
415  || (PreloadedPrivateBufferReg == AMDGPU::NoRegister)) {
416  assert(!ST.isAmdCodeObjectV2(MF));
417  const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
418 
419  unsigned Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
420  unsigned Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
421 
422  // Use relocations to get the pointer, and setup the other bits manually.
423  uint64_t Rsrc23 = TII->getScratchRsrcWords23();
424 
425  if (MFI->hasImplicitBufferPtr()) {
426  unsigned Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
427 
429  const MCInstrDesc &Mov64 = TII->get(AMDGPU::S_MOV_B64);
430 
431  BuildMI(MBB, I, DL, Mov64, Rsrc01)
433  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
434  } else {
435  const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
436 
437  PointerType *PtrTy =
440  MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
441  auto MMO = MF.getMachineMemOperand(PtrInfo,
445  0, 0);
446  BuildMI(MBB, I, DL, LoadDwordX2, Rsrc01)
448  .addImm(0) // offset
449  .addImm(0) // glc
450  .addMemOperand(MMO)
451  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
452  }
453  } else {
454  unsigned Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
455  unsigned Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
456 
457  BuildMI(MBB, I, DL, SMovB32, Rsrc0)
458  .addExternalSymbol("SCRATCH_RSRC_DWORD0")
459  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
460 
461  BuildMI(MBB, I, DL, SMovB32, Rsrc1)
462  .addExternalSymbol("SCRATCH_RSRC_DWORD1")
463  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
464 
465  }
466 
467  BuildMI(MBB, I, DL, SMovB32, Rsrc2)
468  .addImm(Rsrc23 & 0xffffffff)
469  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
470 
471  BuildMI(MBB, I, DL, SMovB32, Rsrc3)
472  .addImm(Rsrc23 >> 32)
473  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
474  }
475 }
476 
478  MachineBasicBlock &MBB) const {
479  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
480  if (FuncInfo->isEntryFunction()) {
481  emitEntryFunctionPrologue(MF, MBB);
482  return;
483  }
484 
485  const MachineFrameInfo &MFI = MF.getFrameInfo();
486  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
487  const SIInstrInfo *TII = ST.getInstrInfo();
488 
489  unsigned StackPtrReg = FuncInfo->getStackPtrOffsetReg();
490  unsigned FramePtrReg = FuncInfo->getFrameOffsetReg();
491 
492  MachineBasicBlock::iterator MBBI = MBB.begin();
493  DebugLoc DL;
494 
495  bool NeedFP = hasFP(MF);
496  if (NeedFP) {
497  // If we need a base pointer, set it up here. It's whatever the value of
498  // the stack pointer is at this point. Any variable size objects will be
499  // allocated after this, so we can still use the base pointer to reference
500  // locals.
501  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
502  .addReg(StackPtrReg)
504  }
505 
506  uint32_t NumBytes = MFI.getStackSize();
507  if (NumBytes != 0 && hasSP(MF)) {
508  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_U32), StackPtrReg)
509  .addReg(StackPtrReg)
510  .addImm(NumBytes * ST.getWavefrontSize())
511  .setMIFlag(MachineInstr::FrameSetup);
512  }
513 
515  : FuncInfo->getSGPRSpillVGPRs()) {
516  if (!Reg.FI.hasValue())
517  continue;
518  TII->storeRegToStackSlot(MBB, MBBI, Reg.VGPR, true,
519  Reg.FI.getValue(), &AMDGPU::VGPR_32RegClass,
520  &TII->getRegisterInfo());
521  }
522 }
523 
525  MachineBasicBlock &MBB) const {
526  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
527  if (FuncInfo->isEntryFunction())
528  return;
529 
530  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
531  const SIInstrInfo *TII = ST.getInstrInfo();
533 
535  : FuncInfo->getSGPRSpillVGPRs()) {
536  if (!Reg.FI.hasValue())
537  continue;
538  TII->loadRegFromStackSlot(MBB, MBBI, Reg.VGPR,
539  Reg.FI.getValue(), &AMDGPU::VGPR_32RegClass,
540  &TII->getRegisterInfo());
541  }
542 
543  unsigned StackPtrReg = FuncInfo->getStackPtrOffsetReg();
544  if (StackPtrReg == AMDGPU::NoRegister)
545  return;
546 
547  const MachineFrameInfo &MFI = MF.getFrameInfo();
548  uint32_t NumBytes = MFI.getStackSize();
549 
550  DebugLoc DL;
551 
552  // FIXME: Clarify distinction between no set SP and SP. For callee functions,
553  // it's really whether we need SP to be accurate or not.
554 
555  if (NumBytes != 0 && hasSP(MF)) {
556  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_SUB_U32), StackPtrReg)
557  .addReg(StackPtrReg)
558  .addImm(NumBytes * ST.getWavefrontSize())
559  .setMIFlag(MachineInstr::FrameDestroy);
560  }
561 }
562 
563 static bool allStackObjectsAreDead(const MachineFrameInfo &MFI) {
564  for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
565  I != E; ++I) {
566  if (!MFI.isDeadObjectIndex(I))
567  return false;
568  }
569 
570  return true;
571 }
572 
574  unsigned &FrameReg) const {
575  const SIRegisterInfo *RI = MF.getSubtarget<SISubtarget>().getRegisterInfo();
576 
577  FrameReg = RI->getFrameRegister(MF);
578  return MF.getFrameInfo().getObjectOffset(FI);
579 }
580 
582  MachineFunction &MF,
583  RegScavenger *RS) const {
584  MachineFrameInfo &MFI = MF.getFrameInfo();
585 
586  if (!MFI.hasStackObjects())
587  return;
588 
589  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
590  const SIInstrInfo *TII = ST.getInstrInfo();
591  const SIRegisterInfo &TRI = TII->getRegisterInfo();
593  bool AllSGPRSpilledToVGPRs = false;
594 
595  if (TRI.spillSGPRToVGPR() && FuncInfo->hasSpilledSGPRs()) {
596  AllSGPRSpilledToVGPRs = true;
597 
598  // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
599  // are spilled to VGPRs, in which case we can eliminate the stack usage.
600  //
601  // XXX - This operates under the assumption that only other SGPR spills are
602  // users of the frame index. I'm not 100% sure this is correct. The
603  // StackColoring pass has a comment saying a future improvement would be to
604  // merging of allocas with spill slots, but for now according to
605  // MachineFrameInfo isSpillSlot can't alias any other object.
606  for (MachineBasicBlock &MBB : MF) {
608  for (auto I = MBB.begin(), E = MBB.end(); I != E; I = Next) {
609  MachineInstr &MI = *I;
610  Next = std::next(I);
611 
612  if (TII->isSGPRSpill(MI)) {
613  int FI = TII->getNamedOperand(MI, AMDGPU::OpName::addr)->getIndex();
614  if (FuncInfo->allocateSGPRSpillToVGPR(MF, FI)) {
615  bool Spilled = TRI.eliminateSGPRToVGPRSpillFrameIndex(MI, FI, RS);
616  (void)Spilled;
617  assert(Spilled && "failed to spill SGPR to VGPR when allocated");
618  } else
619  AllSGPRSpilledToVGPRs = false;
620  }
621  }
622  }
623 
624  FuncInfo->removeSGPRToVGPRFrameIndices(MFI);
625  }
626 
627  // FIXME: The other checks should be redundant with allStackObjectsAreDead,
628  // but currently hasNonSpillStackObjects is set only from source
629  // allocas. Stack temps produced from legalization are not counted currently.
630  if (FuncInfo->hasNonSpillStackObjects() || FuncInfo->hasSpilledVGPRs() ||
631  !AllSGPRSpilledToVGPRs || !allStackObjectsAreDead(MFI)) {
632  assert(RS && "RegScavenger required if spilling");
633 
634  // We force this to be at offset 0 so no user object ever has 0 as an
635  // address, so we may use 0 as an invalid pointer value. This is because
636  // LLVM assumes 0 is an invalid pointer in address space 0. Because alloca
637  // is required to be address space 0, we are forced to accept this for
638  // now. Ideally we could have the stack in another address space with 0 as a
639  // valid pointer, and -1 as the null value.
640  //
641  // This will also waste additional space when user stack objects require > 4
642  // byte alignment.
643  //
644  // The main cost here is losing the offset for addressing modes. However
645  // this also ensures we shouldn't need a register for the offset when
646  // emergency scavenging.
647  int ScavengeFI = MFI.CreateFixedObject(
648  TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0, false);
649  RS->addScavengingFrameIndex(ScavengeFI);
650  }
651 }
652 
654  RegScavenger *RS) const {
657 
658  // The SP is specifically managed and we don't want extra spills of it.
659  SavedRegs.reset(MFI->getStackPtrOffsetReg());
660 }
661 
663  MachineFunction &MF,
664  MachineBasicBlock &MBB,
665  MachineBasicBlock::iterator I) const {
666  int64_t Amount = I->getOperand(0).getImm();
667  if (Amount == 0)
668  return MBB.erase(I);
669 
670  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
671  const SIInstrInfo *TII = ST.getInstrInfo();
672  const DebugLoc &DL = I->getDebugLoc();
673  unsigned Opc = I->getOpcode();
674  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
675  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
676 
678  if (!TFI->hasReservedCallFrame(MF)) {
679  unsigned Align = getStackAlignment();
680 
681  Amount = alignTo(Amount, Align);
682  assert(isUInt<32>(Amount) && "exceeded stack address space size");
684  unsigned SPReg = MFI->getStackPtrOffsetReg();
685 
686  unsigned Op = IsDestroy ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
687  BuildMI(MBB, I, DL, TII->get(Op), SPReg)
688  .addReg(SPReg)
689  .addImm(Amount * ST.getWavefrontSize());
690  } else if (CalleePopAmount != 0) {
691  llvm_unreachable("is this used?");
692  }
693 
694  return MBB.erase(I);
695 }
696 
697 void SIFrameLowering::emitDebuggerPrologue(MachineFunction &MF,
698  MachineBasicBlock &MBB) const {
699  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
700  const SIInstrInfo *TII = ST.getInstrInfo();
701  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
703 
705  DebugLoc DL;
706 
707  // For each dimension:
708  for (unsigned i = 0; i < 3; ++i) {
709  // Get work group ID SGPR, and make it live-in again.
710  unsigned WorkGroupIDSGPR = MFI->getWorkGroupIDSGPR(i);
711  MF.getRegInfo().addLiveIn(WorkGroupIDSGPR);
712  MBB.addLiveIn(WorkGroupIDSGPR);
713 
714  // Since SGPRs are spilled into VGPRs, copy work group ID SGPR to VGPR in
715  // order to spill it to scratch.
716  unsigned WorkGroupIDVGPR =
717  MF.getRegInfo().createVirtualRegister(&AMDGPU::VGPR_32RegClass);
718  BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_e32), WorkGroupIDVGPR)
719  .addReg(WorkGroupIDSGPR);
720 
721  // Spill work group ID.
722  int WorkGroupIDObjectIdx = MFI->getDebuggerWorkGroupIDStackObjectIndex(i);
723  TII->storeRegToStackSlot(MBB, I, WorkGroupIDVGPR, false,
724  WorkGroupIDObjectIdx, &AMDGPU::VGPR_32RegClass, TRI);
725 
726  // Get work item ID VGPR, and make it live-in again.
727  unsigned WorkItemIDVGPR = MFI->getWorkItemIDVGPR(i);
728  MF.getRegInfo().addLiveIn(WorkItemIDVGPR);
729  MBB.addLiveIn(WorkItemIDVGPR);
730 
731  // Spill work item ID.
732  int WorkItemIDObjectIdx = MFI->getDebuggerWorkItemIDStackObjectIndex(i);
733  TII->storeRegToStackSlot(MBB, I, WorkItemIDVGPR, false,
734  WorkItemIDObjectIdx, &AMDGPU::VGPR_32RegClass, TRI);
735  }
736 }
737 
739  // All stack operations are relative to the frame offset SGPR.
740  // TODO: Still want to eliminate sometimes.
741  const MachineFrameInfo &MFI = MF.getFrameInfo();
742 
743  // XXX - Is this only called after frame is finalized? Should be able to check
744  // frame size.
745  return MFI.hasStackObjects() && !allStackObjectsAreDead(MFI);
746 }
747 
749  // All stack operations are relative to the frame offset SGPR.
750  const MachineFrameInfo &MFI = MF.getFrameInfo();
751  return MFI.hasCalls() || MFI.hasVarSizedObjects();
752 }
constexpr bool isUInt< 32 >(uint64_t x)
Definition: MathExtras.h:341
unsigned getWorkItemIDVGPR(unsigned Dim) const
int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const override
getFrameIndexReference - This method should return the base register and offset used to reference a f...
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
Interface definition for SIRegisterInfo.
unsigned reservedPrivateSegmentBufferReg(const MachineFunction &MF) const
Return the end register initially reserved for the scratch buffer in case spilling is needed...
AMDGPU specific subclass of TargetSubtarget.
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:235
bool isAllocatable(unsigned PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn&#39;t been...
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool hasStackObjects() const
Return true if there are any stack objects in this function.
void addLiveIn(unsigned Reg, unsigned vreg=0)
addLiveIn - Add the specified register as a live-in.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS=nullptr) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
void removeSGPRToVGPRFrameIndices(MachineFrameInfo &MFI)
const SIInstrInfo * getInstrInfo() const override
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space...
Definition: Type.cpp:617
void emitEntryFunctionPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const
A debug info location.
Definition: DebugLoc.h:34
uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:677
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:177
const SIRegisterInfo & getRegisterInfo() const
Definition: SIInstrInfo.h:146
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int getDebuggerWorkGroupIDStackObjectIndex(unsigned Dim) const
bool flatScratchIsPointer() const
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
const HexagonInstrInfo * TII
uint64_t getScratchRsrcWords23() const
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
static bool allStackObjectsAreDead(const MachineFrameInfo &MFI)
Reg
All possible values of the reg field in the ModR/M byte.
The memory access is dereferenceable (i.e., doesn&#39;t trap).
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
int getObjectIndexBegin() const
Return the minimum frame object index.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
static ArrayRef< MCPhysReg > getAllSGPR128(const SISubtarget &ST, const MachineFunction &MF)
unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const
bool debuggerEmitPrologue() const
bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI)
Reserve a slice of a VGPR to support spilling for FrameIndex FI.
Class to represent pointers.
Definition: DerivedTypes.h:467
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1164
bool isCompute(CallingConv::ID cc)
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
This file declares the machine register scavenger class.
unsigned const MachineRegisterInfo * MRI
unsigned getFrameRegister(const MachineFunction &MF) const override
unsigned reservedPrivateSegmentWaveByteOffsetReg(const MachineFunction &MF) const
Return the end register initially reserved for the scratch wave offset in case spilling is needed...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:149
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
BitVector & reset()
Definition: BitVector.h:439
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool Immutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:194
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1320
ArrayRef< SGPRSpillVGPRCSR > getSGPRSpillVGPRs() const
void setScratchWaveOffsetReg(unsigned Reg)
const MachineBasicBlock & front() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
virtual bool hasReservedCallFrame(const MachineFunction &MF) const
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required, we reserve argument space for call sites in the function immediately on entry to the current function.
unsigned getWavefrontSize() const
static ArrayRef< MCPhysReg > getAllSGPRs(const SISubtarget &ST, const MachineFunction &MF)
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:194
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
unsigned getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses...
Information about stack frame layout on the target.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
ArrayRef< T > drop_back(size_t N=1) const
Drop the last N elements of the array.
Definition: ArrayRef.h:194
bool isPhysRegUsed(unsigned PhysReg) const
Return true if the specified register is modified or read in this function.
bool eliminateSGPRToVGPRSpillFrameIndex(MachineBasicBlock::iterator MI, int FI, RegScavenger *RS) const
Special case of eliminateFrameIndex.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array...
Definition: ArrayRef.h:179
void replaceRegWith(unsigned FromReg, unsigned ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Address space for constant memory (VTX2)
Definition: AMDGPU.h:225
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
Representation of each machine instruction.
Definition: MachineInstr.h:59
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Interface definition for SIInstrInfo.
unsigned getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned char TargetFlags=0) const
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned getImplicitBufferPtrUserSGPR() const
virtual const TargetFrameLowering * getFrameLowering() const
The memory access always returns the same value (or traps).
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool spillSGPRToVGPR() const
bool hasSP(const MachineFunction &MF) const
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
bool isAmdCodeObjectV2(const MachineFunction &MF) const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isMesaGfxShader(const MachineFunction &MF) const
bool hasSGPRInitBug() const
IRTranslator LLVM IR MI
int getDebuggerWorkItemIDStackObjectIndex(unsigned Dim) const
unsigned getWorkGroupIDSGPR(unsigned Dim) const
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects...
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register...
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
bool hasCalls() const
Return true if the current function has any function calls.