41 UserSGPRInfo(
F, *STI), WorkGroupIDX(
false), WorkGroupIDY(
false),
43 PrivateSegmentWaveByteOffset(
false), WorkItemIDX(
false),
45 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
47 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(
F);
48 WavesPerEU = ST.getWavesPerEU(
F);
49 MaxNumWorkGroups = ST.getMaxNumWorkGroups(
F);
55 VRegFlags.reserve(1024);
67 MayNeedAGPRs = ST.hasMAIInsts();
74 StackPtrOffsetReg = AMDGPU::SGPR32;
76 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
81 ImplicitArgPtr =
false;
87 FrameOffsetReg = AMDGPU::SGPR33;
88 StackPtrOffsetReg = AMDGPU::SGPR32;
90 if (!ST.enableFlatScratch()) {
93 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
99 if (!
F.hasFnAttribute(
"amdgpu-no-implicitarg-ptr"))
100 ImplicitArgPtr =
true;
102 ImplicitArgPtr =
false;
106 if (ST.hasGFX90AInsts() &&
107 ST.getMaxNumVGPRs(
F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
109 MayNeedAGPRs =
false;
114 ST.hasArchitectedSGPRs())) {
115 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workgroup-id-x"))
118 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-y"))
121 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-z"))
126 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workitem-id-x"))
129 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-y") &&
130 ST.getMaxWorkitemID(
F, 1) != 0)
133 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-z") &&
134 ST.getMaxWorkitemID(
F, 2) != 0)
137 if (!IsKernel && !
F.hasFnAttribute(
"amdgpu-no-lds-kernel-id"))
147 if (!ST.flatScratchIsArchitected()) {
148 PrivateSegmentWaveByteOffset =
true;
153 ArgInfo.PrivateSegmentWaveByteOffset =
158 Attribute A =
F.getFnAttribute(
"amdgpu-git-ptr-high");
163 A =
F.getFnAttribute(
"amdgpu-32bit-address-high-bits");
164 S =
A.getValueAsString();
171 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
173 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(
F) - 1);
195 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
197 return ArgInfo.PrivateSegmentBuffer.getRegister();
202 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
204 return ArgInfo.DispatchPtr.getRegister();
209 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
211 return ArgInfo.QueuePtr.getRegister();
217 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
219 return ArgInfo.KernargSegmentPtr.getRegister();
224 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
226 return ArgInfo.DispatchID.getRegister();
231 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
233 return ArgInfo.FlatScratchInit.getRegister();
238 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
240 return ArgInfo.ImplicitBufferPtr.getRegister();
246 return ArgInfo.LDSKernelId.getRegister();
251 unsigned AllocSizeDWord,
int KernArgIdx,
int PaddingSGPRs) {
253 "Preload kernel argument allocated twice.");
254 NumUserSGPRs += PaddingSGPRs;
259 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);
261 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
262 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
263 NumUserSGPRs += AllocSizeDWord;
265 for (
unsigned I = 0;
I < AllocSizeDWord; ++
I) {
266 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
273 return &
ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
290 WWMSpills.
insert(std::make_pair(
300 for (
auto &Reg : WWMSpills) {
302 CalleeSavedRegs.push_back(Reg);
304 ScratchRegs.push_back(Reg);
310 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
311 if (CSRegs[
I] == Reg)
322 for (
unsigned I = 0, E = SpillPhysVGPRs.size();
I < E; ++
I) {
325 TRI->findUnusedRegister(
MRI, &AMDGPU::VGPR_32RegClass, MF);
326 if (!NewReg || NewReg >= Reg)
329 MRI.replaceRegWith(Reg, NewReg);
332 SpillPhysVGPRs[
I] = NewReg;
333 WWMReservedRegs.
remove(Reg);
334 WWMReservedRegs.
insert(NewReg);
335 WWMSpills.
insert(std::make_pair(NewReg, WWMSpills[Reg]));
336 WWMSpills.
erase(Reg);
345bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
350 LaneVGPR =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
351 SpillVGPRs.push_back(LaneVGPR);
353 LaneVGPR = SpillVGPRs.back();
356 SGPRSpillsToVirtualVGPRLanes[FI].push_back(
361bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
362 MachineFunction &MF,
int FI,
unsigned LaneIndex,
bool IsPrologEpilog) {
371 LaneVGPR =
TRI->findUnusedRegister(
MRI, &AMDGPU::VGPR_32RegClass, MF,
373 if (LaneVGPR == AMDGPU::NoRegister) {
376 SGPRSpillsToPhysicalVGPRLanes.erase(FI);
386 SpillPhysVGPRs.push_back(LaneVGPR);
388 LaneVGPR = SpillPhysVGPRs.back();
391 SGPRSpillsToPhysicalVGPRLanes[FI].push_back(
398 bool IsPrologEpilog) {
399 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
400 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
401 : SGPRSpillsToVirtualVGPRLanes[FI];
404 if (!SpillLanes.empty())
409 unsigned WaveSize = ST.getWavefrontSize();
411 unsigned Size = FrameInfo.getObjectSize(FI);
412 unsigned NumLanes =
Size / 4;
414 if (NumLanes > WaveSize)
417 assert(
Size >= 4 &&
"invalid sgpr spill size");
418 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
419 "not spilling SGPRs to VGPRs");
421 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
422 : NumVirtualVGPRSpillLanes;
424 for (
unsigned I = 0;
I < NumLanes; ++
I, ++NumSpillLanes) {
425 unsigned LaneIndex = (NumSpillLanes % WaveSize);
427 bool Allocated = SpillToPhysVGPRLane
428 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
430 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
452 auto &Spill = VGPRToAGPRSpills[FI];
455 if (!Spill.Lanes.empty())
456 return Spill.FullyAllocated;
459 unsigned NumLanes =
Size / 4;
460 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
463 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
466 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
468 Spill.FullyAllocated =
true;
483 OtherUsedRegs.
set(Reg);
485 OtherUsedRegs.
set(Reg);
488 for (
int I = NumLanes - 1;
I >= 0; --
I) {
489 NextSpillReg = std::find_if(
491 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
495 if (NextSpillReg == Regs.
end()) {
496 Spill.FullyAllocated =
false;
500 OtherUsedRegs.
set(*NextSpillReg);
502 MRI.reserveReg(*NextSpillReg,
TRI);
503 Spill.Lanes[
I] = *NextSpillReg++;
506 return Spill.FullyAllocated;
519 SGPRSpillsToVirtualVGPRLanes.erase(R.first);
524 if (!ResetSGPRSpillStackIDs) {
527 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
530 bool HaveSGPRToMemory =
false;
532 if (ResetSGPRSpillStackIDs) {
540 HaveSGPRToMemory =
true;
546 for (
auto &R : VGPRToAGPRSpills) {
551 return HaveSGPRToMemory;
561 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass),
false);
565MCPhysReg SIMachineFunctionInfo::getNextUserSGPR()
const {
566 assert(NumSystemSGPRs == 0 &&
"System SGPRs must be added after user SGPRs");
567 return AMDGPU::SGPR0 + NumUserSGPRs;
570MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR()
const {
571 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
574void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(
Register Reg) {
578void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(
Register NewReg,
580 VRegFlags.grow(NewReg);
581 VRegFlags[NewReg] = VRegFlags[SrcReg];
587 if (!ST.isAmdPalOS())
590 if (ST.hasMergedShaders()) {
596 GitPtrLo = AMDGPU::SGPR8;
615static std::optional<yaml::SIArgumentInfo>
620 auto convertArg = [&](std::optional<yaml::SIArgument> &
A,
627 if (Arg.isRegister()) {
634 SA.
Mask = Arg.getMask();
655 ArgInfo.PrivateSegmentWaveByteOffset);
671 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
672 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
673 GDSSize(MFI.getGDSSize()),
674 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
675 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
676 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
677 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
678 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
679 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
680 Occupancy(MFI.getOccupancy()),
684 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
685 ReturnsVoid(MFI.returnsVoid()),
687 PSInputAddr(MFI.getPSInputAddr()),
688 PSInputEnable(MFI.getPSInputEnable()),
689 Mode(MFI.getMode()) {
740 "", std::nullopt, std::nullopt);
741 SourceRange = YamlMFI.
ScavengeFI->SourceRange;
752 return !
F.hasFnAttribute(
"amdgpu-no-agpr");
759 if (!mayNeedAGPRs()) {
772 for (
unsigned I = 0, E =
MRI.getNumVirtRegs();
I != E; ++
I) {
778 }
else if (!RC && !
MRI.use_empty(Reg) &&
MRI.getType(Reg).isValid()) {
784 for (
MCRegister Reg : AMDGPU::AGPR_32RegClass) {
785 if (
MRI.isPhysRegUsed(Reg)) {
unsigned const MachineRegisterInfo * MRI
Provides AMDGPU specific target descriptions.
Base class for AMDGPU specific classes of TargetSubtarget.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
AMD GCN specific subclass of TargetSubtarget.
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
static std::optional< yaml::SIArgumentInfo > convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, const TargetRegisterInfo &TRI)
static yaml::StringValue regToString(Register Reg, const TargetRegisterInfo &TRI)
Interface definition for SIRegisterInfo.
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
uint32_t getLDSSize() const
bool isChainFunction() const
bool isEntryFunction() const
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Allocate memory in an ever growing pool, as if by bump-pointer.
Lightweight error class with error context and mandatory checking.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
const SITargetLowering * getTargetLowering() const override
void allocKernargPreloadSGPRs(unsigned NumSGPRs)
Wrapper class representing physical registers. Should be passed by value.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
void removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Remove the specified register from the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void setStackID(int ObjectIdx, uint8_t ID)
bool isSpillSlotObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a spill slot.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int getObjectIndexBegin() const
Return the minimum frame object index.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * cloneInfo(const Ty &Old)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
size_type count(const KeyT &Key) const
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This interface provides simple read-only access to a block of memory, and provides simple methods for...
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool usesAGPRs(const MachineFunction &MF) const
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
Register addDispatchPtr(const SIRegisterInfo &TRI)
Register getLongBranchReservedReg() const
Register addFlatScratchInit(const SIRegisterInfo &TRI)
unsigned getMaxWavesPerEU() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
Register addQueuePtr(const SIRegisterInfo &TRI)
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI)=default
Register getGITPtrLoReg(const MachineFunction &MF) const
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
Register getSGPRForEXECCopy() const
bool mayUseAGPRs(const Function &F) const
bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const
void shiftSpillPhysVGPRsToLowestRange(MachineFunction &MF)
Register addLDSKernelId()
Register getVGPRForAGPRCopy() const
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool SpillToPhysVGPRLane=false, bool IsPrologEpilog=false)
Register addKernargSegmentPtr(const SIRegisterInfo &TRI)
Register addDispatchID(const SIRegisterInfo &TRI)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
MachineFunctionInfo * clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, const DenseMap< MachineBasicBlock *, MachineBasicBlock * > &Src2DstMBB) const override
Make a functionally equivalent copy of this MachineFunctionInfo in MF.
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI)
const ReservedRegSet & getWWMReservedRegs() const
std::optional< int > getOptionalScavengeFI() const
Register addImplicitBufferPtr(const SIRegisterInfo &TRI)
void limitOccupancy(const MachineFunction &MF)
SmallVectorImpl< MCRegister > * addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC, unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs)
void reserveWWMRegister(Register Reg)
static bool isChainScratchRegister(Register VGPR)
static bool isAGPRClass(const TargetRegisterClass *RC)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Represents a location in source code.
Represents a range in source code.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
typename SuperClass::const_iterator const_iterator
unsigned getMainFileID() const
const MemoryBuffer * getMemoryBuffer(unsigned i) const
StringRef - Represent a constant reference to a string, i.e.
bool consumeInteger(unsigned Radix, T &Result)
Parse the current string as an integer of the specified radix.
constexpr bool empty() const
empty - Check if the string is empty.
const TargetMachine & getTargetMachine() const
ArrayRef< MCPhysReg > getRegisters() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A raw_ostream that writes to an std::string.
bool isEntryFunctionCC(CallingConv::ID CC)
bool isChainCC(CallingConv::ID CC)
unsigned getInitialPSInputAddr(const Function &F)
bool isGraphics(CallingConv::ID cc)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SPIR_KERNEL
Used for SPIR kernel functions.
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
A serializaable representation of a reference to a stack object or fixed stack object.
std::optional< SIArgument > PrivateSegmentWaveByteOffset
std::optional< SIArgument > WorkGroupIDY
std::optional< SIArgument > FlatScratchInit
std::optional< SIArgument > DispatchPtr
std::optional< SIArgument > DispatchID
std::optional< SIArgument > WorkItemIDY
std::optional< SIArgument > WorkGroupIDX
std::optional< SIArgument > ImplicitArgPtr
std::optional< SIArgument > QueuePtr
std::optional< SIArgument > WorkGroupInfo
std::optional< SIArgument > LDSKernelId
std::optional< SIArgument > ImplicitBufferPtr
std::optional< SIArgument > WorkItemIDX
std::optional< SIArgument > KernargSegmentPtr
std::optional< SIArgument > WorkItemIDZ
std::optional< SIArgument > PrivateSegmentSize
std::optional< SIArgument > PrivateSegmentBuffer
std::optional< SIArgument > WorkGroupIDZ
std::optional< unsigned > Mask
static SIArgument createArgument(bool IsReg)
StringValue SGPRForEXECCopy
SmallVector< StringValue > WWMReservedRegs
uint32_t HighBitsOf32BitAddress
SIMachineFunctionInfo()=default
StringValue LongBranchReservedReg
uint64_t ExplicitKernArgSize
void mappingImpl(yaml::IO &YamlIO) override
StringValue VGPRForAGPRCopy
std::optional< FrameIndex > ScavengeFI
unsigned BytesInStackArgArea
A wrapper around std::string which contains a source range that's being set during parsing.