25#include "llvm/IR/IntrinsicsAMDGPU.h"
27#define DEBUG_TYPE "amdgpu-call-lowering"
48 : OutgoingValueHandler(
B,
MRI), MIB(MIB) {}
65 Register ExtReg = extendRegisterMin32(*
this, ValVReg, VA);
72 if (
TRI->isSGPRReg(
MRI, PhysReg)) {
73 LLT Ty =
MRI.getType(ExtReg);
80 ExtReg = MIRBuilder.buildPtrToInt(S32, ExtReg).getReg(0);
82 ExtReg = MIRBuilder.buildBitcast(S32, ExtReg).getReg(0);
85 auto ToSGPR = MIRBuilder
86 .buildIntrinsic(Intrinsic::amdgcn_readfirstlane,
87 {
MRI.getType(ExtReg)})
89 ExtReg = ToSGPR.getReg(0);
92 MIRBuilder.buildCopy(PhysReg, ExtReg);
101 : IncomingValueHandler(
B,
MRI) {}
106 auto &MFI = MIRBuilder.getMF().getFrameInfo();
110 const bool IsImmutable = !Flags.isByVal();
111 int FI = MFI.CreateFixedObject(
Size,
Offset, IsImmutable);
113 auto AddrReg = MIRBuilder.buildFrameIndex(
115 StackUsed = std::max(StackUsed,
Size +
Offset);
116 return AddrReg.getReg(0);
121 markPhysRegUsed(PhysReg);
126 auto Copy = MIRBuilder.buildCopy(
LLT::scalar(32), PhysReg);
132 MIRBuilder.buildTrunc(ValVReg, Extended);
146 MIRBuilder.buildLoad(ValVReg,
Addr, *MMO);
152 virtual void markPhysRegUsed(
unsigned PhysReg) = 0;
157 : AMDGPUIncomingArgHandler(
B,
MRI) {}
159 void markPhysRegUsed(
unsigned PhysReg)
override {
160 MIRBuilder.getMBB().addLiveIn(PhysReg);
167 : AMDGPUIncomingArgHandler(MIRBuilder,
MRI), MIB(MIB) {}
169 void markPhysRegUsed(
unsigned PhysReg)
override {
176struct AMDGPUOutgoingArgHandler :
public AMDGPUOutgoingValueHandler {
188 bool IsTailCall =
false,
int FPDiff = 0)
189 : AMDGPUOutgoingValueHandler(MIRBuilder,
MRI, MIB), FPDiff(FPDiff),
190 IsTailCall(IsTailCall) {}
204 return FIReg.getReg(0);
211 if (ST.enableFlatScratch()) {
219 SPReg = MIRBuilder.
buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {PtrTy},
226 auto AddrReg = MIRBuilder.
buildPtrAdd(PtrTy, SPReg, OffsetReg);
228 return AddrReg.getReg(0);
234 Register ExtReg = extendRegisterMin32(*
this, ValVReg, VA);
254 ? extendRegister(Arg.
Regs[ValRegIndex], VA)
255 : Arg.
Regs[ValRegIndex];
256 assignValueToAddress(ValVReg,
Addr, MemTy, MPO, VA);
268 case TargetOpcode::G_SEXT:
270 case TargetOpcode::G_ZEXT:
272 case TargetOpcode::G_ANYEXT:
282 bool IsVarArg)
const {
289 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
303 auto &MF =
B.getMF();
315 "For each split Type there should be exactly one VReg.");
319 for (
unsigned i = 0; i < SplitEVTs.
size(); ++i) {
320 EVT VT = SplitEVTs[i];
326 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
327 if (RetInfo.Flags[0].isSExt()) {
328 assert(RetInfo.Regs.size() == 1 &&
"expect only simple return values");
329 ExtendOp = TargetOpcode::G_SEXT;
330 }
else if (RetInfo.Flags[0].isZExt()) {
331 assert(RetInfo.Regs.size() == 1 &&
"expect only simple return values");
332 ExtendOp = TargetOpcode::G_ZEXT;
344 if (Reg != RetInfo.Regs[0]) {
345 RetInfo.Regs[0] =
Reg;
355 OutgoingValueAssigner Assigner(AssignFn);
356 AMDGPUOutgoingValueHandler RetHandler(
B, *
MRI, Ret);
369 assert(!Val == VRegs.
empty() &&
"Return value without a vreg");
373 const bool IsWaveEnd =
376 B.buildInstr(AMDGPU::S_ENDPGM)
382 IsShader ? AMDGPU::SI_RETURN_TO_EPILOG : AMDGPU::SI_RETURN;
383 auto Ret =
B.buildInstrNoInsert(ReturnOpc);
387 else if (!lowerReturnVal(
B, Val, VRegs, Ret))
403 Register KernArgSegmentVReg =
MRI.getLiveInVirtReg(KernArgSegmentPtr);
407 B.buildPtrAdd(DstReg, KernArgSegmentVReg, OffsetReg);
412 Align Alignment)
const {
425 for (
ArgInfo &SplitArg : SplitArgs) {
426 Register PtrReg =
B.getMRI()->createGenericVirtualRegister(PtrTy);
427 lowerParameterPtr(PtrReg,
B,
Offset + FieldOffsets[
Idx]);
430 if (SplitArg.Flags[0].isPointer()) {
444 assert(SplitArg.Regs.size() == 1);
446 B.buildLoad(SplitArg.Regs[0], PtrReg, *MMO);
461 MF.
addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
467 MF.
addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
475 MF.
addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
483 Register VReg =
MRI.createGenericVirtualRegister(P4);
484 MRI.addLiveIn(InputPtrReg, VReg);
485 B.getMBB().addLiveIn(InputPtrReg);
486 B.buildCopy(VReg, InputPtrReg);
492 MF.
addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
498 MF.
addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
518 CCState CCInfo(
F.getCallingConv(),
F.isVarArg(), MF, ArgLocs,
F.getContext());
523 const Align KernArgBaseAlign(16);
528 for (
auto &Arg :
F.args()) {
529 const bool IsByRef = Arg.hasByRefAttr();
530 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
531 unsigned AllocSize =
DL.getTypeAllocSize(ArgTy);
535 MaybeAlign ParamAlign = IsByRef ? Arg.getParamAlign() : std::nullopt;
536 Align ABIAlign =
DL.getValueOrABITypeAlignment(ParamAlign, ArgTy);
538 uint64_t ArgOffset =
alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;
539 ExplicitArgOffset =
alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;
541 if (Arg.use_empty()) {
549 unsigned ByRefAS = cast<PointerType>(Arg.getType())->getAddressSpace();
552 "expected only one register for byval pointers");
554 lowerParameterPtr(VRegs[i][0],
B, ArgOffset);
557 Register PtrReg =
MRI.createGenericVirtualRegister(ConstPtrTy);
558 lowerParameterPtr(PtrReg,
B, ArgOffset);
560 B.buildAddrSpaceCast(VRegs[i][0], PtrReg);
563 ArgInfo OrigArg(VRegs[i], Arg, i);
566 lowerParameter(
B, OrigArg, ArgOffset, Alignment);
600 CCState CCInfo(
CC,
F.isVarArg(), MF, ArgLocs,
F.getContext());
605 MF.
addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
612 MF.
addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
618 unsigned PSInputNum = 0;
625 for (
auto &Arg :
F.args()) {
626 if (
DL.getTypeStoreSize(Arg.getType()) == 0)
629 const bool InReg = Arg.hasAttribute(Attribute::InReg);
632 if (!IsGraphics && InReg)
635 if (Arg.hasAttribute(Attribute::SwiftSelf) ||
636 Arg.hasAttribute(Attribute::SwiftError) ||
637 Arg.hasAttribute(Attribute::Nest))
641 const bool ArgUsed = !Arg.use_empty();
642 bool SkipArg = !ArgUsed && !
Info->isPSInputAllocated(PSInputNum);
645 Info->markPSInputAllocated(PSInputNum);
647 Info->markPSInputEnabled(PSInputNum);
683 if ((
Info->getPSInputAddr() & 0x7F) == 0 ||
684 ((
Info->getPSInputAddr() & 0xF) == 0 &&
685 Info->isPSInputAllocated(11))) {
688 Info->markPSInputAllocated(0);
689 Info->markPSInputEnabled(0);
692 if (Subtarget.isAmdPalOS()) {
701 unsigned PsInputBits =
Info->getPSInputAddr() &
Info->getPSInputEnable();
702 if ((PsInputBits & 0x7F) == 0 ||
703 ((PsInputBits & 0xF) == 0 &&
704 (PsInputBits >> 11 & 1)))
715 if (!IsEntryFunc && !IsGraphics) {
734 if (!Subtarget.enableFlatScratch())
743 Info->setBytesInStackArgArea(StackSize);
784 "amdgpu-no-dispatch-ptr",
785 "amdgpu-no-queue-ptr",
786 "amdgpu-no-implicitarg-ptr",
787 "amdgpu-no-dispatch-id",
788 "amdgpu-no-workgroup-id-x",
789 "amdgpu-no-workgroup-id-y",
790 "amdgpu-no-workgroup-id-z",
791 "amdgpu-no-lds-kernel-id",
801 for (
auto InputID : InputRegs) {
807 if (
Info.CB->hasFnAttr(ImplicitAttrNames[
I++]))
810 std::tie(OutgoingArg, ArgRC, ArgTy) =
817 std::tie(IncomingArg, IncomingArgRC, ArgTy) =
818 CallerArgInfo.getPreloadedValue(InputID);
819 assert(IncomingArgRC == ArgRC);
821 Register InputReg =
MRI.createGenericVirtualRegister(ArgTy);
824 LI->loadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);
826 LI->getImplicitArgPtr(InputReg,
MRI, MIRBuilder);
828 std::optional<uint32_t> Id =
842 ArgRegs.emplace_back(OutgoingArg->
getRegister(), InputReg);
846 LLVM_DEBUG(
dbgs() <<
"Unhandled stack passed implicit input argument\n");
857 std::tie(OutgoingArg, ArgRC, ArgTy) =
860 std::tie(OutgoingArg, ArgRC, ArgTy) =
863 std::tie(OutgoingArg, ArgRC, ArgTy) =
875 const ArgDescriptor *IncomingArgX = std::get<0>(WorkitemIDX);
876 const ArgDescriptor *IncomingArgY = std::get<0>(WorkitemIDY);
877 const ArgDescriptor *IncomingArgZ = std::get<0>(WorkitemIDZ);
880 const bool NeedWorkItemIDX = !
Info.CB->hasFnAttr(
"amdgpu-no-workitem-id-x");
881 const bool NeedWorkItemIDY = !
Info.CB->hasFnAttr(
"amdgpu-no-workitem-id-y");
882 const bool NeedWorkItemIDZ = !
Info.CB->hasFnAttr(
"amdgpu-no-workitem-id-z");
889 if (ST.getMaxWorkitemID(MF.
getFunction(), 0) != 0) {
890 InputReg =
MRI.createGenericVirtualRegister(S32);
891 LI->loadInputValue(InputReg, MIRBuilder, IncomingArgX,
892 std::get<1>(WorkitemIDX), std::get<2>(WorkitemIDX));
899 NeedWorkItemIDY && ST.getMaxWorkitemID(MF.
getFunction(), 1) != 0) {
901 LI->loadInputValue(
Y, MIRBuilder, IncomingArgY, std::get<1>(WorkitemIDY),
902 std::get<2>(WorkitemIDY));
905 InputReg = InputReg ? MIRBuilder.
buildOr(S32, InputReg,
Y).
getReg(0) :
Y;
909 NeedWorkItemIDZ && ST.getMaxWorkitemID(MF.
getFunction(), 2) != 0) {
910 Register Z =
MRI.createGenericVirtualRegister(S32);
911 LI->loadInputValue(Z, MIRBuilder, IncomingArgZ, std::get<1>(WorkitemIDZ),
912 std::get<2>(WorkitemIDZ));
915 InputReg = InputReg ? MIRBuilder.
buildOr(S32, InputReg, Z).
getReg(0) : Z;
919 (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
920 InputReg =
MRI.createGenericVirtualRegister(S32);
921 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
931 IncomingArgX ? *IncomingArgX :
932 IncomingArgY ? *IncomingArgY : *IncomingArgZ, ~0u);
933 LI->loadInputValue(InputReg, MIRBuilder, &IncomingArg,
934 &AMDGPU::VGPR_32RegClass, S32);
940 ArgRegs.emplace_back(OutgoingArg->
getRegister(), InputReg);
945 LLVM_DEBUG(
dbgs() <<
"Unhandled stack passed implicit input argument\n");
954static std::pair<CCAssignFn *, CCAssignFn *>
961 assert(!(IsIndirect && IsTailCall) &&
"Indirect calls can't be tail calls, "
962 "because the address can be divergent");
964 return AMDGPU::G_SI_CALL;
974 if (
Info.Callee.isReg()) {
977 }
else if (
Info.Callee.isGlobal() &&
Info.Callee.getOffset() == 0) {
999 if (CalleeCC == CallerCC)
1005 auto TRI = ST.getRegisterInfo();
1007 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
1008 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
1009 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
1016 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
1021 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
1027 CalleeAssignFnVarArg);
1029 CallerAssignFnVarArg);
1037 if (OutArgs.
empty())
1062 LLVM_DEBUG(
dbgs() <<
"... Cannot fit call operands on caller's stack.\n");
1069 const uint32_t *CallerPreservedMask =
TRI->getCallPreservedMask(MF, CallerCC);
1094 if (!
Info.IsTailCall)
1099 if (
Info.Callee.isReg())
1108 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
1111 if (!CallerPreserved)
1115 LLVM_DEBUG(
dbgs() <<
"... Calling convention cannot be tail called.\n");
1120 return A.hasByValAttr() || A.hasSwiftErrorAttr();
1122 LLVM_DEBUG(
dbgs() <<
"... Cannot tail call from callers with byval "
1123 "or swifterror arguments\n");
1136 <<
"... Caller and callee have incompatible calling conventions.\n");
1143 LLVM_DEBUG(
dbgs() <<
"... Call is eligible for tail call optimization.\n");
1153 ArrayRef<std::pair<MCRegister, Register>> ImplicitArgRegs)
const {
1154 if (!ST.enableFlatScratch()) {
1159 MIRBuilder.
buildCopy(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
1163 for (std::pair<MCRegister, Register> ArgReg : ImplicitArgRegs) {
1190 CallSeqStart = MIRBuilder.
buildInstr(AMDGPU::ADJCALLSTACKUP);
1203 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CalleeCC);
1204 MIB.addRegMask(Mask);
1216 unsigned NumBytes = 0;
1221 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1223 CCState OutInfo(CalleeCC,
false, MF, OutLocs,
F.getContext());
1237 FPDiff = NumReusableBytes - NumBytes;
1245 "unaligned stack on tail call");
1268 AMDGPUOutgoingArgHandler Handler(MIRBuilder,
MRI, MIB,
true, FPDiff);
1277 MIB->getOperand(1).setImm(FPDiff);
1295 if (MIB->getOperand(0).isReg()) {
1297 MF, *
TRI,
MRI, *ST.getInstrInfo(), *ST.getRegBankInfo(), *MIB,
1298 MIB->getDesc(), MIB->getOperand(0), 0));
1302 Info.LoweredTailCall =
true;
1308 if (
Info.IsVarArg) {
1323 for (
auto &OrigArg :
Info.OrigArgs)
1327 if (
Info.CanLowerReturn && !
Info.OrigRet.Ty->isVoidTy())
1331 bool CanTailCallOpt =
1335 if (
Info.IsMustTailCall && !CanTailCallOpt) {
1336 LLVM_DEBUG(
dbgs() <<
"Failed to lower musttail call as tail call\n");
1340 Info.IsTailCall = CanTailCallOpt;
1347 std::tie(AssignFnFixed, AssignFnVarArg) =
1350 MIRBuilder.
buildInstr(AMDGPU::ADJCALLSTACKUP)
1359 MIB.
addDef(
TRI->getReturnAddressReg(MF));
1361 if (!
Info.IsConvergent)
1369 MIB.addRegMask(Mask);
1392 AMDGPUOutgoingArgHandler Handler(MIRBuilder,
MRI, MIB,
false);
1409 if (MIB->getOperand(1).isReg()) {
1411 MF, *
TRI,
MRI, *ST.getInstrInfo(),
1412 *ST.getRegBankInfo(), *MIB, MIB->getDesc(), MIB->getOperand(1),
1422 if (
Info.CanLowerReturn && !
Info.OrigRet.Ty->isVoidTy()) {
1432 uint64_t CalleePopBytes = NumBytes;
1434 MIRBuilder.
buildInstr(AMDGPU::ADJCALLSTACKDOWN)
1438 if (!
Info.CanLowerReturn) {
1440 Info.DemoteRegister,
Info.DemoteStackIndex);
unsigned const MachineRegisterInfo * MRI
static std::pair< CCAssignFn *, CCAssignFn * > getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI)
Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for CC.
static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect, bool IsTailCall)
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool addCallTargetOperands(MachineInstrBuilder &CallInst, MachineIRBuilder &MIRBuilder, AMDGPUCallLowering::CallLoweringInfo &Info)
static ISD::NodeType extOpcodeToISDExtOpcode(unsigned MIOpc)
static void allocateHSAUserSGPRs(CCState &CCInfo, MachineIRBuilder &B, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
This file describes how to lower LLVM calls to machine code calls.
This file declares the targeting of the Machinelegalizer class for AMDGPU.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Interface definition for SIRegisterInfo.
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
bool lowerTailCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info, SmallVectorImpl< ArgInfo > &OutArgs) const
bool isEligibleForTailCallOptimization(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info, SmallVectorImpl< ArgInfo > &InArgs, SmallVectorImpl< ArgInfo > &OutArgs) const
Returns true if the call can be lowered as a tail call.
bool lowerFormalArgumentsKernel(MachineIRBuilder &B, const Function &F, ArrayRef< ArrayRef< Register > > VRegs) const
bool lowerReturn(MachineIRBuilder &B, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI) const override
This hook behaves as the extended lowerReturn function, but for targets that do not support swifterro...
bool areCalleeOutgoingArgsTailCallable(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl< ArgInfo > &OutArgs) const
AMDGPUCallLowering(const AMDGPUTargetLowering &TLI)
bool passSpecialInputs(MachineIRBuilder &MIRBuilder, CCState &CCInfo, SmallVectorImpl< std::pair< MCRegister, Register > > &ArgRegs, CallLoweringInfo &Info) const
bool lowerFormalArguments(MachineIRBuilder &B, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const override
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const override
This hook must be implemented to lower the given call instruction, including argument and return valu...
bool doCallerAndCalleePassArgsTheSameWay(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl< ArgInfo > &InArgs) const
void handleImplicitCallArguments(MachineIRBuilder &MIRBuilder, MachineInstrBuilder &CallInst, const GCNSubtarget &ST, const SIMachineFunctionInfo &MFI, ArrayRef< std::pair< MCRegister, Register > > ImplicitArgRegs) const
This class provides the information for the target register banks.
static std::optional< uint32_t > getLDSKernelIdMetadata(const Function &F)
unsigned getExplicitKernelArgOffset() const
Returns the offset in bytes from the start of the input buffer of the first explicit kernel argument.
EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType ExtendKind) const override
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
Selects the correct CCAssignFn for a given CallingConvention value.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
CCState - This class holds information needed while lowering arguments and return values.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
CCValAssign - Represent assignment of one arg/retval to a location.
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
This class represents a function call, abstracting a target machine's calling convention.
bool handleAssignments(ValueHandler &Handler, SmallVectorImpl< ArgInfo > &Args, CCState &CCState, SmallVectorImpl< CCValAssign > &ArgLocs, MachineIRBuilder &MIRBuilder, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Use Handler to insert code to handle the argument/return values represented by Args.
void insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg, int FI) const
Load the returned value from the stack into virtual registers in VRegs.
bool determineAndHandleAssignments(ValueHandler &Handler, ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, bool IsVarArg, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Invoke ValueAssigner::assignArg on each of the given Args and then use Handler to move them to the as...
bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl< ArgInfo > &InArgs, ValueAssigner &CalleeAssigner, ValueAssigner &CallerAssigner) const
void splitToValueTypes(const ArgInfo &OrigArgInfo, SmallVectorImpl< ArgInfo > &SplitArgs, const DataLayout &DL, CallingConv::ID CallConv, SmallVectorImpl< uint64_t > *Offsets=nullptr) const
Break OrigArgInfo into one or more pieces the calling convention can process, returned in SplitArgs.
void insertSRetIncomingArgument(const Function &F, SmallVectorImpl< ArgInfo > &SplitArgs, Register &DemoteReg, MachineRegisterInfo &MRI, const DataLayout &DL) const
Insert the hidden sret ArgInfo to the beginning of SplitArgs.
void insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg) const
Store the return value given by VRegs into stack starting at the offset specified in DemoteReg.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< ArgInfo > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
bool determineAssignments(ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, CCState &CCInfo) const
Analyze the argument list in Args, using Assigner to populate CCInfo.
bool checkReturn(CCState &CCInfo, SmallVectorImpl< BaseArgInfo > &Outs, CCAssignFn *Fn) const
void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const
A parsed version of the target data layout string in and methods for querying it.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
iterator_range< arg_iterator > args()
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
const SIRegisterInfo * getRegisterInfo() const override
bool hasKernargSegmentPtr() const
bool hasDispatchID() const
bool hasPrivateSegmentBuffer() const
bool hasImplicitBufferPtr() const
bool hasDispatchPtr() const
bool hasFlatScratchInit() const
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
constexpr unsigned getScalarSizeInBits() const
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr ElementCount getElementCount() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
This is an important class for using LLVM in a threaded context.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setHasTailCall(bool V=true)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
Helper class to build MachineInstr.
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
MachineInstrBuilder buildGlobalValue(const DstOp &Res, const GlobalValue *GV)
Build and insert Res = G_GLOBAL_VALUE GV.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
A Module instance is used to store all the information related to an LLVM module.
Wrapper class representing virtual and physical registers.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Register getStackPtrOffsetReg() const
Register getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses.
unsigned getBytesInStackArgArea() const
void setIfReturnsVoid(bool Value)
MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const
AMDGPUFunctionArgInfo & getArgInfo()
void allocateSpecialInputSGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocateSpecialInputVGPRsFixed(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
Allocate implicit function VGPR arguments in fixed registers.
void allocateSpecialEntryInputVGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF, SIMachineFunctionInfo &Info, CallingConv::ID CallConv, bool IsShader) const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ PRIVATE_ADDRESS
Address space for private memory.
unsigned getCodeObjectVersion(const Module &M)
bool isEntryFunctionCC(CallingConv::ID CC)
LLVM_READNONE bool isKernel(CallingConv::ID CC)
bool isShader(CallingConv::ID cc)
bool isGraphics(CallingConv::ID cc)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ SIGN_EXTEND
Conversion operators.
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< TypeSize > *Offsets, TypeSize StartingOffset)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
Align inferAlignFromPtrInfo(MachineFunction &MF, const MachinePointerInfo &MPO)
ArgDescriptor WorkItemIDZ
ArgDescriptor WorkItemIDY
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(PreloadedValue Value) const
ArgDescriptor WorkItemIDX
This struct is a compact representation of a valid (non-zero power of two) alignment.
MCRegister getRegister() const
static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask)
Helper struct shared between Function Specialization and SCCP Solver.
SmallVector< Register, 4 > Regs
Base class for ValueHandlers used for arguments coming into the current function, or for return value...
Register buildExtensionHint(CCValAssign &VA, Register SrcReg, LLT NarrowTy)
Insert G_ASSERT_ZEXT/G_ASSERT_SEXT or other hint instruction based on VA, returning the new register ...
void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign VA) override
Provides a default implementation for argument handling.
Base class for ValueHandlers used for arguments passed to a function call, or for return values.
uint64_t StackSize
The size of the currently allocated portion of the stack.
MachineIRBuilder & MIRBuilder
Register extendRegister(Register ValReg, CCValAssign &VA, unsigned MaxSizeBits=0)
Extend a register to the location type given in VA, capped at extending to at most MaxSize bits.
virtual Register getStackAddress(uint64_t MemSize, int64_t Offset, MachinePointerInfo &MPO, ISD::ArgFlagsTy Flags)=0
Materialize a VReg containing the address of the specified stack-based object.
virtual void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy, MachinePointerInfo &MPO, CCValAssign &VA)=0
The specified value has been assigned to a stack location.
virtual void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign VA)=0
The specified value has been assigned to a physical register, handle the appropriate COPY (either to ...
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.