41 UserSGPRInfo(
F, *STI), WorkGroupIDX(
false), WorkGroupIDY(
false),
43 PrivateSegmentWaveByteOffset(
false), WorkItemIDX(
false),
45 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
47 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(
F);
48 WavesPerEU = ST.getWavesPerEU(
F);
53 VRegFlags.reserve(1024);
65 MayNeedAGPRs = ST.hasMAIInsts();
72 StackPtrOffsetReg = AMDGPU::SGPR32;
74 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
79 ImplicitArgPtr =
false;
85 FrameOffsetReg = AMDGPU::SGPR33;
86 StackPtrOffsetReg = AMDGPU::SGPR32;
88 if (!ST.enableFlatScratch()) {
91 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
97 if (!
F.hasFnAttribute(
"amdgpu-no-implicitarg-ptr"))
98 ImplicitArgPtr =
true;
100 ImplicitArgPtr =
false;
104 if (ST.hasGFX90AInsts() &&
105 ST.getMaxNumVGPRs(
F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
107 MayNeedAGPRs =
false;
112 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workgroup-id-x"))
115 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-y"))
118 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-z"))
123 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workitem-id-x"))
126 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-y") &&
127 ST.getMaxWorkitemID(
F, 1) != 0)
130 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-z") &&
131 ST.getMaxWorkitemID(
F, 2) != 0)
134 if (!IsKernel && !
F.hasFnAttribute(
"amdgpu-no-lds-kernel-id"))
144 if (!ST.flatScratchIsArchitected()) {
145 PrivateSegmentWaveByteOffset =
true;
150 ArgInfo.PrivateSegmentWaveByteOffset =
155 Attribute A =
F.getFnAttribute(
"amdgpu-git-ptr-high");
160 A =
F.getFnAttribute(
"amdgpu-32bit-address-high-bits");
161 S =
A.getValueAsString();
168 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
170 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(
F) - 1);
192 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
194 return ArgInfo.PrivateSegmentBuffer.getRegister();
199 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
201 return ArgInfo.DispatchPtr.getRegister();
206 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
208 return ArgInfo.QueuePtr.getRegister();
214 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
216 return ArgInfo.KernargSegmentPtr.getRegister();
221 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
223 return ArgInfo.DispatchID.getRegister();
228 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
230 return ArgInfo.FlatScratchInit.getRegister();
235 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
237 return ArgInfo.ImplicitBufferPtr.getRegister();
243 return ArgInfo.LDSKernelId.getRegister();
252 WWMSpills.
insert(std::make_pair(
262 for (
auto &Reg : WWMSpills) {
264 CalleeSavedRegs.push_back(Reg);
266 ScratchRegs.push_back(Reg);
272 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
273 if (CSRegs[
I] == Reg)
280bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
285 LaneVGPR =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
286 SpillVGPRs.push_back(LaneVGPR);
288 LaneVGPR = SpillVGPRs.back();
291 SGPRSpillsToVirtualVGPRLanes[FI].push_back(
296bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
303 LaneVGPR =
TRI->findUnusedRegister(
MRI, &AMDGPU::VGPR_32RegClass, MF);
304 if (LaneVGPR == AMDGPU::NoRegister) {
307 SGPRSpillsToPhysicalVGPRLanes.erase(FI);
318 LaneVGPR = WWMReservedRegs.
back();
321 SGPRSpillsToPhysicalVGPRLanes[FI].push_back(
328 bool IsPrologEpilog) {
329 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
330 IsPrologEpilog ? SGPRSpillsToPhysicalVGPRLanes[FI]
331 : SGPRSpillsToVirtualVGPRLanes[FI];
334 if (!SpillLanes.empty())
339 unsigned WaveSize = ST.getWavefrontSize();
341 unsigned Size = FrameInfo.getObjectSize(FI);
342 unsigned NumLanes =
Size / 4;
344 if (NumLanes > WaveSize)
347 assert(
Size >= 4 &&
"invalid sgpr spill size");
348 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
349 "not spilling SGPRs to VGPRs");
351 unsigned &NumSpillLanes =
352 IsPrologEpilog ? NumPhysicalVGPRSpillLanes : NumVirtualVGPRSpillLanes;
354 for (
unsigned I = 0;
I < NumLanes; ++
I, ++NumSpillLanes) {
355 unsigned LaneIndex = (NumSpillLanes % WaveSize);
357 bool Allocated = IsPrologEpilog
358 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex)
359 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
381 auto &Spill = VGPRToAGPRSpills[FI];
384 if (!Spill.Lanes.empty())
385 return Spill.FullyAllocated;
388 unsigned NumLanes =
Size / 4;
389 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
392 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
395 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
397 Spill.FullyAllocated =
true;
412 OtherUsedRegs.
set(Reg);
414 OtherUsedRegs.
set(Reg);
417 for (
int I = NumLanes - 1;
I >= 0; --
I) {
418 NextSpillReg = std::find_if(
420 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
424 if (NextSpillReg == Regs.
end()) {
425 Spill.FullyAllocated =
false;
429 OtherUsedRegs.
set(*NextSpillReg);
431 MRI.reserveReg(*NextSpillReg,
TRI);
432 Spill.Lanes[
I] = *NextSpillReg++;
435 return Spill.FullyAllocated;
448 SGPRSpillsToVirtualVGPRLanes.erase(R.first);
453 if (!ResetSGPRSpillStackIDs) {
456 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
459 bool HaveSGPRToMemory =
false;
461 if (ResetSGPRSpillStackIDs) {
469 HaveSGPRToMemory =
true;
475 for (
auto &R : VGPRToAGPRSpills) {
480 return HaveSGPRToMemory;
489 TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0,
false);
492 TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
493 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass),
false);
498MCPhysReg SIMachineFunctionInfo::getNextUserSGPR()
const {
499 assert(NumSystemSGPRs == 0 &&
"System SGPRs must be added after user SGPRs");
500 return AMDGPU::SGPR0 + NumUserSGPRs;
503MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR()
const {
504 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
507void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(
Register Reg) {
511void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(
Register NewReg,
513 VRegFlags.grow(NewReg);
514 VRegFlags[NewReg] = VRegFlags[SrcReg];
520 if (!ST.isAmdPalOS())
523 if (ST.hasMergedShaders()) {
529 GitPtrLo = AMDGPU::SGPR8;
548static std::optional<yaml::SIArgumentInfo>
553 auto convertArg = [&](std::optional<yaml::SIArgument> &
A,
560 if (Arg.isRegister()) {
567 SA.
Mask = Arg.getMask();
587 ArgInfo.PrivateSegmentWaveByteOffset);
603 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
604 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
605 GDSSize(MFI.getGDSSize()),
606 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
607 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
608 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
609 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
610 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
611 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
612 Occupancy(MFI.getOccupancy()),
616 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
617 ReturnsVoid(MFI.returnsVoid()),
619 PSInputAddr(MFI.getPSInputAddr()),
620 PSInputEnable(MFI.getPSInputEnable()),
621 Mode(MFI.getMode()) {
672 "", std::nullopt, std::nullopt);
673 SourceRange = YamlMFI.
ScavengeFI->SourceRange;
686 const auto *CB = dyn_cast<CallBase>(&
I);
690 if (CB->isInlineAsm()) {
691 const InlineAsm *IA = dyn_cast<InlineAsm>(CB->getCalledOperand());
692 for (
const auto &CI : IA->ParseConstraints()) {
694 Code.consume_front(
"{");
695 if (Code.startswith(
"a"))
703 dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
719 if (!mayNeedAGPRs()) {
732 for (
unsigned I = 0,
E =
MRI.getNumVirtRegs();
I !=
E; ++
I) {
738 }
else if (!RC && !
MRI.use_empty(Reg) &&
MRI.getType(Reg).isValid()) {
744 for (
MCRegister Reg : AMDGPU::AGPR_32RegClass) {
745 if (
MRI.isPhysRegUsed(Reg)) {
unsigned const MachineRegisterInfo * MRI
Provides AMDGPU specific target descriptions.
Base class for AMDGPU specific classes of TargetSubtarget.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
static std::optional< yaml::SIArgumentInfo > convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, const TargetRegisterInfo &TRI)
static yaml::StringValue regToString(Register Reg, const TargetRegisterInfo &TRI)
Interface definition for SIRegisterInfo.
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
uint32_t getLDSSize() const
bool isEntryFunction() const
LLVM Basic Block Representation.
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Allocate memory in an ever growing pool, as if by bump-pointer.
Lightweight error class with error context and mandatory checking.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
const SITargetLowering * getTargetLowering() const override
Wrapper class representing physical registers. Should be passed by value.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void setStackID(int ObjectIdx, uint8_t ID)
bool isSpillSlotObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a spill slot.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int getObjectIndexBegin() const
Return the minimum frame object index.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * cloneInfo(const Ty &Old)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
size_type count(const KeyT &Key) const
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This interface provides simple read-only access to a block of memory, and provides simple methods for...
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool usesAGPRs(const MachineFunction &MF) const
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
Register addDispatchPtr(const SIRegisterInfo &TRI)
Register getLongBranchReservedReg() const
Register addFlatScratchInit(const SIRegisterInfo &TRI)
unsigned getMaxWavesPerEU() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
Register addQueuePtr(const SIRegisterInfo &TRI)
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool IsPrologEpilog=false)
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI)=default
Register getGITPtrLoReg(const MachineFunction &MF) const
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
Register getSGPRForEXECCopy() const
bool mayUseAGPRs(const Function &F) const
bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const
Register addLDSKernelId()
Register getVGPRForAGPRCopy() const
Register addKernargSegmentPtr(const SIRegisterInfo &TRI)
Register addDispatchID(const SIRegisterInfo &TRI)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
MachineFunctionInfo * clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, const DenseMap< MachineBasicBlock *, MachineBasicBlock * > &Src2DstMBB) const override
Make a functionally equivalent copy of this MachineFunctionInfo in MF.
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI)
const ReservedRegSet & getWWMReservedRegs() const
std::optional< int > getOptionalScavengeFI() const
Register addImplicitBufferPtr(const SIRegisterInfo &TRI)
void limitOccupancy(const MachineFunction &MF)
void reserveWWMRegister(Register Reg)
static bool isAGPRClass(const TargetRegisterClass *RC)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Represents a location in source code.
Represents a range in source code.
const value_type & back() const
Return the last element of the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
typename SuperClass::const_iterator const_iterator
unsigned getMainFileID() const
const MemoryBuffer * getMemoryBuffer(unsigned i) const
StringRef - Represent a constant reference to a string, i.e.
bool consumeInteger(unsigned Radix, T &Result)
Parse the current string as an integer of the specified radix.
constexpr bool empty() const
empty - Check if the string is empty.
const TargetMachine & getTargetMachine() const
iterator_range< SmallVectorImpl< MCPhysReg >::const_iterator > getRegisters() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A raw_ostream that writes to an std::string.
bool isEntryFunctionCC(CallingConv::ID CC)
bool isChainCC(CallingConv::ID CC)
unsigned getInitialPSInputAddr(const Function &F)
bool isGraphics(CallingConv::ID cc)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SPIR_KERNEL
Used for SPIR kernel functions.
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static constexpr ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
A serializaable representation of a reference to a stack object or fixed stack object.
std::optional< SIArgument > PrivateSegmentWaveByteOffset
std::optional< SIArgument > WorkGroupIDY
std::optional< SIArgument > FlatScratchInit
std::optional< SIArgument > DispatchPtr
std::optional< SIArgument > DispatchID
std::optional< SIArgument > WorkItemIDY
std::optional< SIArgument > WorkGroupIDX
std::optional< SIArgument > ImplicitArgPtr
std::optional< SIArgument > QueuePtr
std::optional< SIArgument > WorkGroupInfo
std::optional< SIArgument > LDSKernelId
std::optional< SIArgument > ImplicitBufferPtr
std::optional< SIArgument > WorkItemIDX
std::optional< SIArgument > KernargSegmentPtr
std::optional< SIArgument > WorkItemIDZ
std::optional< SIArgument > PrivateSegmentSize
std::optional< SIArgument > PrivateSegmentBuffer
std::optional< SIArgument > WorkGroupIDZ
std::optional< unsigned > Mask
static SIArgument createArgument(bool IsReg)
StringValue SGPRForEXECCopy
SmallVector< StringValue > WWMReservedRegs
uint32_t HighBitsOf32BitAddress
SIMachineFunctionInfo()=default
StringValue LongBranchReservedReg
uint64_t ExplicitKernArgSize
void mappingImpl(yaml::IO &YamlIO) override
StringValue VGPRForAGPRCopy
std::optional< FrameIndex > ScavengeFI
unsigned BytesInStackArgArea
A wrapper around std::string which contains a source range that's being set during parsing.