LLVM 20.0.0git
SIMachineFunctionInfo.cpp
Go to the documentation of this file.
1//===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10#include "AMDGPUSubtarget.h"
11#include "GCNSubtarget.h"
13#include "SIRegisterInfo.h"
21#include "llvm/IR/CallingConv.h"
23#include "llvm/IR/Function.h"
24#include <cassert>
25#include <optional>
26#include <vector>
27
28enum { MAX_LANES = 64 };
29
30using namespace llvm;
31
33 const SITargetLowering *TLI = STI->getTargetLowering();
34 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
35}
36
38 const GCNSubtarget *STI)
39 : AMDGPUMachineFunction(F, *STI), Mode(F, *STI), GWSResourcePSV(getTM(STI)),
40 UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false),
41 WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false),
42 PrivateSegmentWaveByteOffset(false), WorkItemIDX(false),
43 WorkItemIDY(false), WorkItemIDZ(false), ImplicitArgPtr(false),
44 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
45 const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI);
46 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
47 WavesPerEU = ST.getWavesPerEU(F);
48 MaxNumWorkGroups = ST.getMaxNumWorkGroups(F);
49 assert(MaxNumWorkGroups.size() == 3);
50
51 Occupancy = ST.computeOccupancy(F, getLDSSize());
52 CallingConv::ID CC = F.getCallingConv();
53
54 VRegFlags.reserve(1024);
55
56 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
58
59 if (IsKernel) {
60 WorkGroupIDX = true;
61 WorkItemIDX = true;
62 } else if (CC == CallingConv::AMDGPU_PS) {
63 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
64 }
65
66 MayNeedAGPRs = ST.hasMAIInsts();
67
68 if (AMDGPU::isChainCC(CC)) {
69 // Chain functions don't receive an SP from their caller, but are free to
70 // set one up. For now, we can use s32 to match what amdgpu_gfx functions
71 // would use if called, but this can be revisited.
72 // FIXME: Only reserve this if we actually need it.
73 StackPtrOffsetReg = AMDGPU::SGPR32;
74
75 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
76
77 ArgInfo.PrivateSegmentBuffer =
78 ArgDescriptor::createRegister(ScratchRSrcReg);
79
80 ImplicitArgPtr = false;
81 } else if (!isEntryFunction()) {
84
85 FrameOffsetReg = AMDGPU::SGPR33;
86 StackPtrOffsetReg = AMDGPU::SGPR32;
87
88 if (!ST.enableFlatScratch()) {
89 // Non-entry functions have no special inputs for now, other registers
90 // required for scratch access.
91 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
92
93 ArgInfo.PrivateSegmentBuffer =
94 ArgDescriptor::createRegister(ScratchRSrcReg);
95 }
96
97 if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
98 ImplicitArgPtr = true;
99 } else {
100 ImplicitArgPtr = false;
101 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
103
104 if (ST.hasGFX90AInsts() &&
105 ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
106 !mayUseAGPRs(F))
107 MayNeedAGPRs = false; // We will select all MAI with VGPR operands.
108 }
109
110 if (!AMDGPU::isGraphics(CC) ||
112 ST.hasArchitectedSGPRs())) {
113 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x"))
114 WorkGroupIDX = true;
115
116 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y"))
117 WorkGroupIDY = true;
118
119 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z"))
120 WorkGroupIDZ = true;
121 }
122
123 if (!AMDGPU::isGraphics(CC)) {
124 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x"))
125 WorkItemIDX = true;
126
127 if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") &&
128 ST.getMaxWorkitemID(F, 1) != 0)
129 WorkItemIDY = true;
130
131 if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") &&
132 ST.getMaxWorkitemID(F, 2) != 0)
133 WorkItemIDZ = true;
134
135 if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id"))
136 LDSKernelId = true;
137 }
138
139 if (isEntryFunction()) {
140 // X, XY, and XYZ are the only supported combinations, so make sure Y is
141 // enabled if Z is.
142 if (WorkItemIDZ)
143 WorkItemIDY = true;
144
145 if (!ST.flatScratchIsArchitected()) {
146 PrivateSegmentWaveByteOffset = true;
147
148 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
149 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
151 ArgInfo.PrivateSegmentWaveByteOffset =
152 ArgDescriptor::createRegister(AMDGPU::SGPR5);
153 }
154 }
155
156 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
157 StringRef S = A.getValueAsString();
158 if (!S.empty())
159 S.consumeInteger(0, GITPtrHigh);
160
161 A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
162 S = A.getValueAsString();
163 if (!S.empty())
164 S.consumeInteger(0, HighBitsOf32BitAddress);
165
166 MaxMemoryClusterDWords = F.getFnAttributeAsParsedInteger(
167 "amdgpu-max-memory-cluster-dwords", DefaultMemoryClusterDWordsLimit);
168
169 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
170 // VGPR available at all times. For now, reserve highest available VGPR. After
171 // RA, shift it to the lowest available unused VGPR if the one exist.
172 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
173 VGPRForAGPRCopy =
174 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1);
175 }
176}
177
179 BumpPtrAllocator &Allocator, MachineFunction &DestMF,
181 const {
182 return DestMF.cloneInfo<SIMachineFunctionInfo>(*this);
183}
184
187 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
188 limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
189 MF.getFunction()));
190}
191
193 const SIRegisterInfo &TRI) {
194 ArgInfo.PrivateSegmentBuffer =
195 ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
196 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
197 NumUserSGPRs += 4;
198 return ArgInfo.PrivateSegmentBuffer.getRegister();
199}
200
202 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
203 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
204 NumUserSGPRs += 2;
205 return ArgInfo.DispatchPtr.getRegister();
206}
207
209 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
210 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
211 NumUserSGPRs += 2;
212 return ArgInfo.QueuePtr.getRegister();
213}
214
216 ArgInfo.KernargSegmentPtr
217 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
218 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
219 NumUserSGPRs += 2;
220 return ArgInfo.KernargSegmentPtr.getRegister();
221}
222
224 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
225 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
226 NumUserSGPRs += 2;
227 return ArgInfo.DispatchID.getRegister();
228}
229
231 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
232 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
233 NumUserSGPRs += 2;
234 return ArgInfo.FlatScratchInit.getRegister();
235}
236
238 ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(getNextUserSGPR());
239 NumUserSGPRs += 1;
240 return ArgInfo.PrivateSegmentSize.getRegister();
241}
242
244 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
245 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
246 NumUserSGPRs += 2;
247 return ArgInfo.ImplicitBufferPtr.getRegister();
248}
249
251 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());
252 NumUserSGPRs += 1;
253 return ArgInfo.LDSKernelId.getRegister();
254}
255
257 const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
258 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {
259 assert(!ArgInfo.PreloadKernArgs.count(KernArgIdx) &&
260 "Preload kernel argument allocated twice.");
261 NumUserSGPRs += PaddingSGPRs;
262 // If the available register tuples are aligned with the kernarg to be
263 // preloaded use that register, otherwise we need to use a set of SGPRs and
264 // merge them.
265 if (!ArgInfo.FirstKernArgPreloadReg)
266 ArgInfo.FirstKernArgPreloadReg = getNextUserSGPR();
267 Register PreloadReg =
268 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);
269 if (PreloadReg &&
270 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
271 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
272 NumUserSGPRs += AllocSizeDWord;
273 } else {
274 for (unsigned I = 0; I < AllocSizeDWord; ++I) {
275 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
276 NumUserSGPRs++;
277 }
278 }
279
280 // Track the actual number of SGPRs that HW will preload to.
281 UserSGPRInfo.allocKernargPreloadSGPRs(AllocSizeDWord + PaddingSGPRs);
282 return &ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
283}
284
286 uint64_t Size, Align Alignment) {
287 // Skip if it is an entry function or the register is already added.
288 if (isEntryFunction() || WWMSpills.count(VGPR))
289 return;
290
291 // Skip if this is a function with the amdgpu_cs_chain or
292 // amdgpu_cs_chain_preserve calling convention and this is a scratch register.
293 // We never need to allocate a spill for these because we don't even need to
294 // restore the inactive lanes for them (they're scratchier than the usual
295 // scratch registers). We only need to do this if we have calls to
296 // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since
297 // chain functions do not return) and the function did not contain a call to
298 // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes
299 // when entering the function).
300 if (isChainFunction() &&
303 return;
304
305 WWMSpills.insert(std::make_pair(
306 VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
307}
308
309// Separate out the callee-saved and scratch registers.
311 MachineFunction &MF,
312 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
313 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
314 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
315 for (auto &Reg : WWMSpills) {
316 if (isCalleeSavedReg(CSRegs, Reg.first))
317 CalleeSavedRegs.push_back(Reg);
318 else
319 ScratchRegs.push_back(Reg);
320 }
321}
322
324 MCPhysReg Reg) const {
325 for (unsigned I = 0; CSRegs[I]; ++I) {
326 if (CSRegs[I] == Reg)
327 return true;
328 }
329
330 return false;
331}
332
335 BitVector &SavedVGPRs) {
336 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
338 for (unsigned I = 0, E = WWMVGPRs.size(); I < E; ++I) {
339 Register Reg = WWMVGPRs[I];
340 Register NewReg =
341 TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
342 if (!NewReg || NewReg >= Reg)
343 break;
344
345 MRI.replaceRegWith(Reg, NewReg);
346
347 // Update various tables with the new VGPR.
348 WWMVGPRs[I] = NewReg;
349 WWMReservedRegs.remove(Reg);
350 WWMReservedRegs.insert(NewReg);
351 MRI.reserveReg(NewReg, TRI);
352
353 // Replace the register in SpillPhysVGPRs. This is needed to look for free
354 // lanes while spilling special SGPRs like FP, BP, etc. during PEI.
355 auto *RegItr = std::find(SpillPhysVGPRs.begin(), SpillPhysVGPRs.end(), Reg);
356 if (RegItr != SpillPhysVGPRs.end()) {
357 unsigned Idx = std::distance(SpillPhysVGPRs.begin(), RegItr);
358 SpillPhysVGPRs[Idx] = NewReg;
359 }
360
361 // The generic `determineCalleeSaves` might have set the old register if it
362 // is in the CSR range.
363 SavedVGPRs.reset(Reg);
364
365 for (MachineBasicBlock &MBB : MF) {
366 MBB.removeLiveIn(Reg);
368 }
369
370 Reg = NewReg;
371 }
372}
373
374bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
375 MachineFunction &MF, int FI, unsigned LaneIndex) {
377 Register LaneVGPR;
378 if (!LaneIndex) {
379 LaneVGPR = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
380 SpillVGPRs.push_back(LaneVGPR);
381 } else {
382 LaneVGPR = SpillVGPRs.back();
383 }
384
385 SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);
386 return true;
387}
388
389bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
390 MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) {
392 const SIRegisterInfo *TRI = ST.getRegisterInfo();
394 Register LaneVGPR;
395 if (!LaneIndex) {
396 // Find the highest available register if called before RA to ensure the
397 // lowest registers are available for allocation. The LaneVGPR, in that
398 // case, will be shifted back to the lowest range after VGPR allocation.
399 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF,
400 !IsPrologEpilog);
401 if (LaneVGPR == AMDGPU::NoRegister) {
402 // We have no VGPRs left for spilling SGPRs. Reset because we will not
403 // partially spill the SGPR to VGPRs.
404 SGPRSpillsToPhysicalVGPRLanes.erase(FI);
405 return false;
406 }
407
408 if (IsPrologEpilog)
409 allocateWWMSpill(MF, LaneVGPR);
410
411 reserveWWMRegister(LaneVGPR);
412 for (MachineBasicBlock &MBB : MF) {
413 MBB.addLiveIn(LaneVGPR);
415 }
416 SpillPhysVGPRs.push_back(LaneVGPR);
417 } else {
418 LaneVGPR = SpillPhysVGPRs.back();
419 }
420
421 SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);
422 return true;
423}
424
426 MachineFunction &MF, int FI, bool SpillToPhysVGPRLane,
427 bool IsPrologEpilog) {
428 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
429 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
430 : SGPRSpillsToVirtualVGPRLanes[FI];
431
432 // This has already been allocated.
433 if (!SpillLanes.empty())
434 return true;
435
436 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
437 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
438 unsigned WaveSize = ST.getWavefrontSize();
439
440 unsigned Size = FrameInfo.getObjectSize(FI);
441 unsigned NumLanes = Size / 4;
442
443 if (NumLanes > WaveSize)
444 return false;
445
446 assert(Size >= 4 && "invalid sgpr spill size");
447 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
448 "not spilling SGPRs to VGPRs");
449
450 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
451 : NumVirtualVGPRSpillLanes;
452
453 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
454 unsigned LaneIndex = (NumSpillLanes % WaveSize);
455
456 bool Allocated = SpillToPhysVGPRLane
457 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
458 IsPrologEpilog)
459 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
460 if (!Allocated) {
461 NumSpillLanes -= I;
462 return false;
463 }
464 }
465
466 return true;
467}
468
469/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
470/// Either AGPR is spilled to VGPR to vice versa.
471/// Returns true if a \p FI can be eliminated completely.
473 int FI,
474 bool isAGPRtoVGPR) {
476 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
477 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
478
479 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
480
481 auto &Spill = VGPRToAGPRSpills[FI];
482
483 // This has already been allocated.
484 if (!Spill.Lanes.empty())
485 return Spill.FullyAllocated;
486
487 unsigned Size = FrameInfo.getObjectSize(FI);
488 unsigned NumLanes = Size / 4;
489 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
490
491 const TargetRegisterClass &RC =
492 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
493 auto Regs = RC.getRegisters();
494
495 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
496 const SIRegisterInfo *TRI = ST.getRegisterInfo();
497 Spill.FullyAllocated = true;
498
499 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
500 // once.
501 BitVector OtherUsedRegs;
502 OtherUsedRegs.resize(TRI->getNumRegs());
503
504 const uint32_t *CSRMask =
505 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
506 if (CSRMask)
507 OtherUsedRegs.setBitsInMask(CSRMask);
508
509 // TODO: Should include register tuples, but doesn't matter with current
510 // usage.
511 for (MCPhysReg Reg : SpillAGPR)
512 OtherUsedRegs.set(Reg);
513 for (MCPhysReg Reg : SpillVGPR)
514 OtherUsedRegs.set(Reg);
515
516 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
517 for (int I = NumLanes - 1; I >= 0; --I) {
518 NextSpillReg = std::find_if(
519 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
520 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
521 !OtherUsedRegs[Reg];
522 });
523
524 if (NextSpillReg == Regs.end()) { // Registers exhausted
525 Spill.FullyAllocated = false;
526 break;
527 }
528
529 OtherUsedRegs.set(*NextSpillReg);
530 SpillRegs.push_back(*NextSpillReg);
531 MRI.reserveReg(*NextSpillReg, TRI);
532 Spill.Lanes[I] = *NextSpillReg++;
533 }
534
535 return Spill.FullyAllocated;
536}
537
539 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
540 // Remove dead frame indices from function frame, however keep FP & BP since
541 // spills for them haven't been inserted yet. And also make sure to remove the
542 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,
543 // otherwise, it could result in an unexpected side effect and bug, in case of
544 // any re-mapping of freed frame indices by later pass(es) like "stack slot
545 // coloring".
546 for (auto &R : make_early_inc_range(SGPRSpillsToVirtualVGPRLanes)) {
547 MFI.RemoveStackObject(R.first);
548 SGPRSpillsToVirtualVGPRLanes.erase(R.first);
549 }
550
551 // Remove the dead frame indices of CSR SGPRs which are spilled to physical
552 // VGPR lanes during SILowerSGPRSpills pass.
553 if (!ResetSGPRSpillStackIDs) {
554 for (auto &R : make_early_inc_range(SGPRSpillsToPhysicalVGPRLanes)) {
555 MFI.RemoveStackObject(R.first);
556 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
557 }
558 }
559 bool HaveSGPRToMemory = false;
560
561 if (ResetSGPRSpillStackIDs) {
562 // All other SGPRs must be allocated on the default stack, so reset the
563 // stack ID.
564 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
565 ++I) {
569 HaveSGPRToMemory = true;
570 }
571 }
572 }
573 }
574
575 for (auto &R : VGPRToAGPRSpills) {
576 if (R.second.IsDead)
577 MFI.RemoveStackObject(R.first);
578 }
579
580 return HaveSGPRToMemory;
581}
582
584 const SIRegisterInfo &TRI) {
585 if (ScavengeFI)
586 return *ScavengeFI;
587
588 ScavengeFI =
589 MFI.CreateStackObject(TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
590 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false);
591 return *ScavengeFI;
592}
593
594MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
595 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
596 return AMDGPU::SGPR0 + NumUserSGPRs;
597}
598
599MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
600 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
601}
602
603void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
604 VRegFlags.grow(Reg);
605}
606
607void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
608 Register SrcReg) {
609 VRegFlags.grow(NewReg);
610 VRegFlags[NewReg] = VRegFlags[SrcReg];
611}
612
615 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
616 if (!ST.isAmdPalOS())
617 return Register();
618 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
619 if (ST.hasMergedShaders()) {
620 switch (MF.getFunction().getCallingConv()) {
623 // Low GIT address is passed in s8 rather than s0 for an LS+HS or
624 // ES+GS merged shader on gfx9+.
625 GitPtrLo = AMDGPU::SGPR8;
626 return GitPtrLo;
627 default:
628 return GitPtrLo;
629 }
630 }
631 return GitPtrLo;
632}
633
635 const TargetRegisterInfo &TRI) {
637 {
639 OS << printReg(Reg, &TRI);
640 }
641 return Dest;
642}
643
644static std::optional<yaml::SIArgumentInfo>
646 const TargetRegisterInfo &TRI) {
648
649 auto convertArg = [&](std::optional<yaml::SIArgument> &A,
650 const ArgDescriptor &Arg) {
651 if (!Arg)
652 return false;
653
654 // Create a register or stack argument.
656 if (Arg.isRegister()) {
658 OS << printReg(Arg.getRegister(), &TRI);
659 } else
660 SA.StackOffset = Arg.getStackOffset();
661 // Check and update the optional mask.
662 if (Arg.isMasked())
663 SA.Mask = Arg.getMask();
664
665 A = SA;
666 return true;
667 };
668
669 // TODO: Need to serialize kernarg preloads.
670 bool Any = false;
671 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
672 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
673 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
674 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
675 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
676 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
677 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
678 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
679 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
680 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
681 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
682 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
683 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
684 ArgInfo.PrivateSegmentWaveByteOffset);
685 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
686 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
687 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
688 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
689 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
690
691 if (Any)
692 return AI;
693
694 return std::nullopt;
695}
696
699 const llvm::MachineFunction &MF)
700 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
701 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
702 GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()),
703 IsEntryFunction(MFI.isEntryFunction()),
704 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
705 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
706 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
707 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
708 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
709 Occupancy(MFI.getOccupancy()),
710 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
711 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
712 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
713 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
714 ReturnsVoid(MFI.returnsVoid()),
715 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
716 PSInputAddr(MFI.getPSInputAddr()), PSInputEnable(MFI.getPSInputEnable()),
717 MaxMemoryClusterDWords(MFI.getMaxMemoryClusterDWords()),
718 Mode(MFI.getMode()) {
719 for (Register Reg : MFI.getSGPRSpillPhysVGPRs())
720 SpillPhysVGPRS.push_back(regToString(Reg, TRI));
721
722 for (Register Reg : MFI.getWWMReservedRegs())
723 WWMReservedRegs.push_back(regToString(Reg, TRI));
724
725 if (MFI.getLongBranchReservedReg())
727 if (MFI.getVGPRForAGPRCopy())
729
730 if (MFI.getSGPRForEXECCopy())
732
733 auto SFI = MFI.getOptionalScavengeFI();
734 if (SFI)
736}
737
740}
741
743 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
747 LDSSize = YamlMFI.LDSSize;
748 GDSSize = YamlMFI.GDSSize;
749 DynLDSAlign = YamlMFI.DynLDSAlign;
750 PSInputAddr = YamlMFI.PSInputAddr;
754 Occupancy = YamlMFI.Occupancy;
757 MemoryBound = YamlMFI.MemoryBound;
758 WaveLimiter = YamlMFI.WaveLimiter;
762 ReturnsVoid = YamlMFI.ReturnsVoid;
763
764 if (YamlMFI.ScavengeFI) {
765 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo());
766 if (!FIOrErr) {
767 // Create a diagnostic for a the frame index.
768 const MemoryBuffer &Buffer =
769 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
770
771 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
772 SourceMgr::DK_Error, toString(FIOrErr.takeError()),
773 "", {}, {});
774 SourceRange = YamlMFI.ScavengeFI->SourceRange;
775 return true;
776 }
777 ScavengeFI = *FIOrErr;
778 } else {
779 ScavengeFI = std::nullopt;
780 }
781 return false;
782}
783
785 return !F.hasFnAttribute("amdgpu-no-agpr");
786}
787
789 if (UsesAGPRs)
790 return *UsesAGPRs;
791
792 if (!mayNeedAGPRs()) {
793 UsesAGPRs = false;
794 return false;
795 }
796
798 MF.getFrameInfo().hasCalls()) {
799 UsesAGPRs = true;
800 return true;
801 }
802
803 const MachineRegisterInfo &MRI = MF.getRegInfo();
804
805 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
807 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
808 if (RC && SIRegisterInfo::isAGPRClass(RC)) {
809 UsesAGPRs = true;
810 return true;
811 }
812 if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) {
813 // Defer caching UsesAGPRs, function might not yet been regbank selected.
814 return true;
815 }
816 }
817
818 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) {
819 if (MRI.isPhysRegUsed(Reg)) {
820 UsesAGPRs = true;
821 return true;
822 }
823 }
824
825 UsesAGPRs = false;
826 return false;
827}
unsigned const MachineRegisterInfo * MRI
Provides AMDGPU specific target descriptions.
Base class for AMDGPU specific classes of TargetSubtarget.
MachineBasicBlock & MBB
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
uint64_t Size
IO & YamlIO
Definition: ELFYAML.cpp:1314
AMD GCN specific subclass of TargetSubtarget.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
static std::optional< yaml::SIArgumentInfo > convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, const TargetRegisterInfo &TRI)
static yaml::StringValue regToString(Register Reg, const TargetRegisterInfo &TRI)
Interface definition for SIRegisterInfo.
raw_pwrite_stream & OS
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
Definition: Any.h:28
BitVector & reset()
Definition: BitVector.h:392
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Definition: BitVector.h:707
void push_back(bool Val)
Definition: BitVector.h:466
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:66
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:277
const SITargetLowering * getTargetLowering() const override
Definition: GCNSubtarget.h:287
void allocKernargPreloadSGPRs(unsigned NumSGPRs)
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
void removeLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Remove the specified register from the live in set.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void setStackID(int ObjectIdx, uint8_t ID)
bool hasTailCall() const
Returns true if the function contains a tail call.
bool isSpillSlotObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a spill slot.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int getObjectIndexBegin() const
Return the minimum frame object index.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * cloneInfo(const Ty &Old)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
size_type count(const KeyT &Key) const
Definition: MapVector.h:165
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: MapVector.h:141
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:51
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Definition: MemoryBuffer.h:76
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
Definition: Register.h:84
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool usesAGPRs(const MachineFunction &MF) const
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void shiftWwmVGPRsToLowestRange(MachineFunction &MF, SmallVectorImpl< Register > &WWMVGPRs, BitVector &SavedVGPRs)
Register addPrivateSegmentSize(const SIRegisterInfo &TRI)
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
Register addDispatchPtr(const SIRegisterInfo &TRI)
Register addFlatScratchInit(const SIRegisterInfo &TRI)
ArrayRef< Register > getSGPRSpillPhysVGPRs() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
Register addQueuePtr(const SIRegisterInfo &TRI)
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI)=default
Register getGITPtrLoReg(const MachineFunction &MF) const
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
bool mayUseAGPRs(const Function &F) const
bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool SpillToPhysVGPRLane=false, bool IsPrologEpilog=false)
Register addKernargSegmentPtr(const SIRegisterInfo &TRI)
Register addDispatchID(const SIRegisterInfo &TRI)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
MachineFunctionInfo * clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, const DenseMap< MachineBasicBlock *, MachineBasicBlock * > &Src2DstMBB) const override
Make a functionally equivalent copy of this MachineFunctionInfo in MF.
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI)
const ReservedRegSet & getWWMReservedRegs() const
std::optional< int > getOptionalScavengeFI() const
Register addImplicitBufferPtr(const SIRegisterInfo &TRI)
void limitOccupancy(const MachineFunction &MF)
SmallVectorImpl< MCRegister > * addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC, unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs)
static bool isChainScratchRegister(Register VGPR)
static bool isAGPRClass(const TargetRegisterClass *RC)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Definition: SourceMgr.h:281
Represents a location in source code.
Definition: SMLoc.h:23
Represents a range in source code.
Definition: SMLoc.h:48
bool remove(const value_type &X)
Remove an item from the set vector.
Definition: SetVector.h:188
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
size_t size() const
Definition: SmallVector.h:78
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
typename SuperClass::const_iterator const_iterator
Definition: SmallVector.h:578
unsigned getMainFileID() const
Definition: SourceMgr.h:132
const MemoryBuffer * getMemoryBuffer(unsigned i) const
Definition: SourceMgr.h:125
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
bool consumeInteger(unsigned Radix, T &Result)
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:499
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:147
const TargetMachine & getTargetMachine() const
ArrayRef< MCPhysReg > getRegisters() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
bool isEntryFunctionCC(CallingConv::ID CC)
bool isChainCC(CallingConv::ID CC)
unsigned getInitialPSInputAddr(const Function &F)
bool isGraphics(CallingConv::ID cc)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
Definition: CallingConv.h:197
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
Definition: CallingConv.h:200
@ AMDGPU_Gfx
Used for AMD graphics targets.
Definition: CallingConv.h:232
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
Definition: CallingConv.h:206
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
Definition: CallingConv.h:191
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
Definition: CallingConv.h:194
@ SPIR_KERNEL
Used for SPIR kernel functions.
Definition: CallingConv.h:144
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:657
constexpr unsigned DefaultMemoryClusterDWordsLimit
Definition: SIInstrInfo.h:39
const char * toString(DWARFSectionKind Kind)
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
Definition: SCCPSolver.h:41
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
A serializaable representation of a reference to a stack object or fixed stack object.
std::optional< SIArgument > PrivateSegmentWaveByteOffset
std::optional< SIArgument > WorkGroupIDY
std::optional< SIArgument > FlatScratchInit
std::optional< SIArgument > DispatchPtr
std::optional< SIArgument > DispatchID
std::optional< SIArgument > WorkItemIDY
std::optional< SIArgument > WorkGroupIDX
std::optional< SIArgument > ImplicitArgPtr
std::optional< SIArgument > QueuePtr
std::optional< SIArgument > WorkGroupInfo
std::optional< SIArgument > LDSKernelId
std::optional< SIArgument > ImplicitBufferPtr
std::optional< SIArgument > WorkItemIDX
std::optional< SIArgument > KernargSegmentPtr
std::optional< SIArgument > WorkItemIDZ
std::optional< SIArgument > PrivateSegmentSize
std::optional< SIArgument > PrivateSegmentBuffer
std::optional< SIArgument > WorkGroupIDZ
std::optional< unsigned > Mask
static SIArgument createArgument(bool IsReg)
SmallVector< StringValue > WWMReservedRegs
void mappingImpl(yaml::IO &YamlIO) override
SmallVector< StringValue, 2 > SpillPhysVGPRS
std::optional< FrameIndex > ScavengeFI
A wrapper around std::string which contains a source range that's being set during parsing.