Bug Summary

File:llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
Warning:line 370, column 25
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AMDGPUCallLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/Target/AMDGPU -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/Target/AMDGPU -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-06-21-164211-33944-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
1//===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUCallLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPULegalizerInfo.h"
18#include "AMDGPUTargetMachine.h"
19#include "SIMachineFunctionInfo.h"
20#include "SIRegisterInfo.h"
21#include "llvm/CodeGen/Analysis.h"
22#include "llvm/CodeGen/FunctionLoweringInfo.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/IR/IntrinsicsAMDGPU.h"
25
26#define DEBUG_TYPE"amdgpu-call-lowering" "amdgpu-call-lowering"
27
28using namespace llvm;
29
30namespace {
31
32/// Wrapper around extendRegister to ensure we extend to a full 32-bit register.
33static Register extendRegisterMin32(CallLowering::ValueHandler &Handler,
34 Register ValVReg, CCValAssign &VA) {
35 if (VA.getLocVT().getSizeInBits() < 32) {
36 // 16-bit types are reported as legal for 32-bit registers. We need to
37 // extend and do a 32-bit copy to avoid the verifier complaining about it.
38 return Handler.MIRBuilder.buildAnyExt(LLT::scalar(32), ValVReg).getReg(0);
39 }
40
41 return Handler.extendRegister(ValVReg, VA);
42}
43
44struct AMDGPUOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
45 AMDGPUOutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
46 MachineInstrBuilder MIB)
47 : OutgoingValueHandler(B, MRI), MIB(MIB) {}
48
49 MachineInstrBuilder MIB;
50
51 Register getStackAddress(uint64_t Size, int64_t Offset,
52 MachinePointerInfo &MPO,
53 ISD::ArgFlagsTy Flags) override {
54 llvm_unreachable("not implemented")::llvm::llvm_unreachable_internal("not implemented", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 54)
;
55 }
56
57 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
58 MachinePointerInfo &MPO, CCValAssign &VA) override {
59 llvm_unreachable("not implemented")::llvm::llvm_unreachable_internal("not implemented", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 59)
;
60 }
61
62 void assignValueToReg(Register ValVReg, Register PhysReg,
63 CCValAssign &VA) override {
64 Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);
65
66 // If this is a scalar return, insert a readfirstlane just in case the value
67 // ends up in a VGPR.
68 // FIXME: Assert this is a shader return.
69 const SIRegisterInfo *TRI
70 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
71 if (TRI->isSGPRReg(MRI, PhysReg)) {
72 auto ToSGPR = MIRBuilder.buildIntrinsic(Intrinsic::amdgcn_readfirstlane,
73 {MRI.getType(ExtReg)}, false)
74 .addReg(ExtReg);
75 ExtReg = ToSGPR.getReg(0);
76 }
77
78 MIRBuilder.buildCopy(PhysReg, ExtReg);
79 MIB.addUse(PhysReg, RegState::Implicit);
80 }
81};
82
83struct AMDGPUIncomingArgHandler : public CallLowering::IncomingValueHandler {
84 uint64_t StackUsed = 0;
85
86 AMDGPUIncomingArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
87 : IncomingValueHandler(B, MRI) {}
88
89 Register getStackAddress(uint64_t Size, int64_t Offset,
90 MachinePointerInfo &MPO,
91 ISD::ArgFlagsTy Flags) override {
92 auto &MFI = MIRBuilder.getMF().getFrameInfo();
93
94 // Byval is assumed to be writable memory, but other stack passed arguments
95 // are not.
96 const bool IsImmutable = !Flags.isByVal();
97 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);
98 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
99 auto AddrReg = MIRBuilder.buildFrameIndex(
100 LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32), FI);
101 StackUsed = std::max(StackUsed, Size + Offset);
102 return AddrReg.getReg(0);
103 }
104
105 void assignValueToReg(Register ValVReg, Register PhysReg,
106 CCValAssign &VA) override {
107 markPhysRegUsed(PhysReg);
108
109 if (VA.getLocVT().getSizeInBits() < 32) {
110 // 16-bit types are reported as legal for 32-bit registers. We need to do
111 // a 32-bit copy, and truncate to avoid the verifier complaining about it.
112 auto Copy = MIRBuilder.buildCopy(LLT::scalar(32), PhysReg);
113
114 // If we have signext/zeroext, it applies to the whole 32-bit register
115 // before truncation.
116 auto Extended =
117 buildExtensionHint(VA, Copy.getReg(0), LLT(VA.getLocVT()));
118 MIRBuilder.buildTrunc(ValVReg, Extended);
119 return;
120 }
121
122 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
123 }
124
125 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t MemSize,
126 MachinePointerInfo &MPO, CCValAssign &VA) override {
127 MachineFunction &MF = MIRBuilder.getMF();
128
129 auto MMO = MF.getMachineMemOperand(
130 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemSize,
131 inferAlignFromPtrInfo(MF, MPO));
132 MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
133 }
134
135 /// How the physical register gets marked varies between formal
136 /// parameters (it's a basic-block live-in), and a call instruction
137 /// (it's an implicit-def of the BL).
138 virtual void markPhysRegUsed(unsigned PhysReg) = 0;
139};
140
141struct FormalArgHandler : public AMDGPUIncomingArgHandler {
142 FormalArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
143 : AMDGPUIncomingArgHandler(B, MRI) {}
144
145 void markPhysRegUsed(unsigned PhysReg) override {
146 MIRBuilder.getMBB().addLiveIn(PhysReg);
147 }
148};
149
150struct CallReturnHandler : public AMDGPUIncomingArgHandler {
151 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
152 MachineInstrBuilder MIB)
153 : AMDGPUIncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
154
155 void markPhysRegUsed(unsigned PhysReg) override {
156 MIB.addDef(PhysReg, RegState::Implicit);
157 }
158
159 MachineInstrBuilder MIB;
160};
161
162struct AMDGPUOutgoingArgHandler : public AMDGPUOutgoingValueHandler {
163 /// For tail calls, the byte offset of the call's argument area from the
164 /// callee's. Unused elsewhere.
165 int FPDiff;
166
167 // Cache the SP register vreg if we need it more than once in this call site.
168 Register SPReg;
169
170 bool IsTailCall;
171
172 AMDGPUOutgoingArgHandler(MachineIRBuilder &MIRBuilder,
173 MachineRegisterInfo &MRI, MachineInstrBuilder MIB,
174 bool IsTailCall = false, int FPDiff = 0)
175 : AMDGPUOutgoingValueHandler(MIRBuilder, MRI, MIB), FPDiff(FPDiff),
176 IsTailCall(IsTailCall) {}
177
178 Register getStackAddress(uint64_t Size, int64_t Offset,
179 MachinePointerInfo &MPO,
180 ISD::ArgFlagsTy Flags) override {
181 MachineFunction &MF = MIRBuilder.getMF();
182 const LLT PtrTy = LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32);
183 const LLT S32 = LLT::scalar(32);
184
185 if (IsTailCall) {
186 Offset += FPDiff;
187 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
188 auto FIReg = MIRBuilder.buildFrameIndex(PtrTy, FI);
189 MPO = MachinePointerInfo::getFixedStack(MF, FI);
190 return FIReg.getReg(0);
191 }
192
193 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
194
195 if (!SPReg)
196 SPReg = MIRBuilder.buildCopy(PtrTy, MFI->getStackPtrOffsetReg()).getReg(0);
197
198 auto OffsetReg = MIRBuilder.buildConstant(S32, Offset);
199
200 auto AddrReg = MIRBuilder.buildPtrAdd(PtrTy, SPReg, OffsetReg);
201 MPO = MachinePointerInfo::getStack(MF, Offset);
202 return AddrReg.getReg(0);
203 }
204
205 void assignValueToReg(Register ValVReg, Register PhysReg,
206 CCValAssign &VA) override {
207 MIB.addUse(PhysReg, RegState::Implicit);
208 Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);
209 MIRBuilder.buildCopy(PhysReg, ExtReg);
210 }
211
212 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
213 MachinePointerInfo &MPO, CCValAssign &VA) override {
214 MachineFunction &MF = MIRBuilder.getMF();
215 uint64_t LocMemOffset = VA.getLocMemOffset();
216 const auto &ST = MF.getSubtarget<GCNSubtarget>();
217
218 auto MMO = MF.getMachineMemOperand(
219 MPO, MachineMemOperand::MOStore, Size,
220 commonAlignment(ST.getStackAlignment(), LocMemOffset));
221 MIRBuilder.buildStore(ValVReg, Addr, *MMO);
222 }
223
224 void assignValueToAddress(const CallLowering::ArgInfo &Arg,
225 unsigned ValRegIndex, Register Addr,
226 uint64_t MemSize, MachinePointerInfo &MPO,
227 CCValAssign &VA) override {
228 Register ValVReg = VA.getLocInfo() != CCValAssign::LocInfo::FPExt
229 ? extendRegister(Arg.Regs[ValRegIndex], VA)
230 : Arg.Regs[ValRegIndex];
231 assignValueToAddress(ValVReg, Addr, MemSize, MPO, VA);
232 }
233};
234}
235
236AMDGPUCallLowering::AMDGPUCallLowering(const AMDGPUTargetLowering &TLI)
237 : CallLowering(&TLI) {
238}
239
240// FIXME: Compatability shim
241static ISD::NodeType extOpcodeToISDExtOpcode(unsigned MIOpc) {
242 switch (MIOpc) {
243 case TargetOpcode::G_SEXT:
244 return ISD::SIGN_EXTEND;
245 case TargetOpcode::G_ZEXT:
246 return ISD::ZERO_EXTEND;
247 case TargetOpcode::G_ANYEXT:
248 return ISD::ANY_EXTEND;
249 default:
250 llvm_unreachable("not an extend opcode")::llvm::llvm_unreachable_internal("not an extend opcode", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 250)
;
251 }
252}
253
254bool AMDGPUCallLowering::canLowerReturn(MachineFunction &MF,
255 CallingConv::ID CallConv,
256 SmallVectorImpl<BaseArgInfo> &Outs,
257 bool IsVarArg) const {
258 // For shaders. Vector types should be explicitly handled by CC.
259 if (AMDGPU::isEntryFunctionCC(CallConv))
260 return true;
261
262 SmallVector<CCValAssign, 16> ArgLocs;
263 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
264 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
265 MF.getFunction().getContext());
266
267 return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv, IsVarArg));
268}
269
270/// Lower the return value for the already existing \p Ret. This assumes that
271/// \p B's insertion point is correct.
272bool AMDGPUCallLowering::lowerReturnVal(MachineIRBuilder &B,
273 const Value *Val, ArrayRef<Register> VRegs,
274 MachineInstrBuilder &Ret) const {
275 if (!Val)
276 return true;
277
278 auto &MF = B.getMF();
279 const auto &F = MF.getFunction();
280 const DataLayout &DL = MF.getDataLayout();
281 MachineRegisterInfo *MRI = B.getMRI();
282 LLVMContext &Ctx = F.getContext();
283
284 CallingConv::ID CC = F.getCallingConv();
285 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
286
287 SmallVector<EVT, 8> SplitEVTs;
288 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
289 assert(VRegs.size() == SplitEVTs.size() &&(static_cast <bool> (VRegs.size() == SplitEVTs.size() &&
"For each split Type there should be exactly one VReg.") ? void
(0) : __assert_fail ("VRegs.size() == SplitEVTs.size() && \"For each split Type there should be exactly one VReg.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 290, __extension__ __PRETTY_FUNCTION__))
290 "For each split Type there should be exactly one VReg.")(static_cast <bool> (VRegs.size() == SplitEVTs.size() &&
"For each split Type there should be exactly one VReg.") ? void
(0) : __assert_fail ("VRegs.size() == SplitEVTs.size() && \"For each split Type there should be exactly one VReg.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 290, __extension__ __PRETTY_FUNCTION__))
;
291
292 SmallVector<ArgInfo, 8> SplitRetInfos;
293
294 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
295 EVT VT = SplitEVTs[i];
296 Register Reg = VRegs[i];
297 ArgInfo RetInfo(Reg, VT.getTypeForEVT(Ctx));
298 setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);
299
300 if (VT.isScalarInteger()) {
301 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
302 if (RetInfo.Flags[0].isSExt()) {
303 assert(RetInfo.Regs.size() == 1 && "expect only simple return values")(static_cast <bool> (RetInfo.Regs.size() == 1 &&
"expect only simple return values") ? void (0) : __assert_fail
("RetInfo.Regs.size() == 1 && \"expect only simple return values\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 303, __extension__ __PRETTY_FUNCTION__))
;
304 ExtendOp = TargetOpcode::G_SEXT;
305 } else if (RetInfo.Flags[0].isZExt()) {
306 assert(RetInfo.Regs.size() == 1 && "expect only simple return values")(static_cast <bool> (RetInfo.Regs.size() == 1 &&
"expect only simple return values") ? void (0) : __assert_fail
("RetInfo.Regs.size() == 1 && \"expect only simple return values\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 306, __extension__ __PRETTY_FUNCTION__))
;
307 ExtendOp = TargetOpcode::G_ZEXT;
308 }
309
310 EVT ExtVT = TLI.getTypeForExtReturn(Ctx, VT,
311 extOpcodeToISDExtOpcode(ExtendOp));
312 if (ExtVT != VT) {
313 RetInfo.Ty = ExtVT.getTypeForEVT(Ctx);
314 LLT ExtTy = getLLTForType(*RetInfo.Ty, DL);
315 Reg = B.buildInstr(ExtendOp, {ExtTy}, {Reg}).getReg(0);
316 }
317 }
318
319 if (Reg != RetInfo.Regs[0]) {
320 RetInfo.Regs[0] = Reg;
321 // Reset the arg flags after modifying Reg.
322 setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);
323 }
324
325 splitToValueTypes(RetInfo, SplitRetInfos, DL, CC);
326 }
327
328 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC, F.isVarArg());
329
330 OutgoingValueAssigner Assigner(AssignFn);
331 AMDGPUOutgoingValueHandler RetHandler(B, *MRI, Ret);
332 return determineAndHandleAssignments(RetHandler, Assigner, SplitRetInfos, B,
333 CC, F.isVarArg());
334}
335
336bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &B, const Value *Val,
337 ArrayRef<Register> VRegs,
338 FunctionLoweringInfo &FLI) const {
339
340 MachineFunction &MF = B.getMF();
341 MachineRegisterInfo &MRI = MF.getRegInfo();
342 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
343 MFI->setIfReturnsVoid(!Val);
1
Assuming 'Val' is null
344
345 assert(!Val == VRegs.empty() && "Return value without a vreg")(static_cast <bool> (!Val == VRegs.empty() && "Return value without a vreg"
) ? void (0) : __assert_fail ("!Val == VRegs.empty() && \"Return value without a vreg\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 345, __extension__ __PRETTY_FUNCTION__))
;
2
Assuming the condition is true
3
'?' condition is true
346
347 CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
348 const bool IsShader = AMDGPU::isShader(CC);
349 const bool IsWaveEnd =
350 (IsShader && MFI->returnsVoid()) || AMDGPU::isKernel(CC);
4
Assuming 'IsShader' is false
351 if (IsWaveEnd
4.1
'IsWaveEnd' is false
) {
5
Taking false branch
352 B.buildInstr(AMDGPU::S_ENDPGM)
353 .addImm(0);
354 return true;
355 }
356
357 auto const &ST = MF.getSubtarget<GCNSubtarget>();
358
359 unsigned ReturnOpc =
360 IsShader
5.1
'IsShader' is false
? AMDGPU::SI_RETURN_TO_EPILOG : AMDGPU::S_SETPC_B64_return;
6
'?' condition is false
361
362 auto Ret = B.buildInstrNoInsert(ReturnOpc);
363 Register ReturnAddrVReg;
364 if (ReturnOpc
6.1
'ReturnOpc' is equal to S_SETPC_B64_return
== AMDGPU::S_SETPC_B64_return) {
7
Taking true branch
365 ReturnAddrVReg = MRI.createVirtualRegister(&AMDGPU::CCR_SGPR_64RegClass);
366 Ret.addUse(ReturnAddrVReg);
367 }
368
369 if (!FLI.CanLowerReturn)
8
Assuming field 'CanLowerReturn' is false
9
Taking true branch
370 insertSRetStores(B, Val->getType(), VRegs, FLI.DemoteRegister);
10
Called C++ object pointer is null
371 else if (!lowerReturnVal(B, Val, VRegs, Ret))
372 return false;
373
374 if (ReturnOpc == AMDGPU::S_SETPC_B64_return) {
375 const SIRegisterInfo *TRI = ST.getRegisterInfo();
376 Register LiveInReturn = MF.addLiveIn(TRI->getReturnAddressReg(MF),
377 &AMDGPU::SGPR_64RegClass);
378 B.buildCopy(ReturnAddrVReg, LiveInReturn);
379 }
380
381 // TODO: Handle CalleeSavedRegsViaCopy.
382
383 B.insertInstr(Ret);
384 return true;
385}
386
387void AMDGPUCallLowering::lowerParameterPtr(Register DstReg, MachineIRBuilder &B,
388 Type *ParamTy,
389 uint64_t Offset) const {
390 MachineFunction &MF = B.getMF();
391 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
392 MachineRegisterInfo &MRI = MF.getRegInfo();
393 Register KernArgSegmentPtr =
394 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
395 Register KernArgSegmentVReg = MRI.getLiveInVirtReg(KernArgSegmentPtr);
396
397 auto OffsetReg = B.buildConstant(LLT::scalar(64), Offset);
398
399 B.buildPtrAdd(DstReg, KernArgSegmentVReg, OffsetReg);
400}
401
402void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, Type *ParamTy,
403 uint64_t Offset, Align Alignment,
404 Register DstReg) const {
405 MachineFunction &MF = B.getMF();
406 const Function &F = MF.getFunction();
407 const DataLayout &DL = F.getParent()->getDataLayout();
408 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
409 unsigned TypeSize = DL.getTypeStoreSize(ParamTy);
410
411 LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
412 Register PtrReg = B.getMRI()->createGenericVirtualRegister(PtrTy);
413 lowerParameterPtr(PtrReg, B, ParamTy, Offset);
414
415 MachineMemOperand *MMO = MF.getMachineMemOperand(
416 PtrInfo,
417 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
418 MachineMemOperand::MOInvariant,
419 TypeSize, Alignment);
420
421 B.buildLoad(DstReg, PtrReg, *MMO);
422}
423
424// Allocate special inputs passed in user SGPRs.
425static void allocateHSAUserSGPRs(CCState &CCInfo,
426 MachineIRBuilder &B,
427 MachineFunction &MF,
428 const SIRegisterInfo &TRI,
429 SIMachineFunctionInfo &Info) {
430 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
431 if (Info.hasPrivateSegmentBuffer()) {
432 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
433 MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
434 CCInfo.AllocateReg(PrivateSegmentBufferReg);
435 }
436
437 if (Info.hasDispatchPtr()) {
438 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
439 MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
440 CCInfo.AllocateReg(DispatchPtrReg);
441 }
442
443 if (Info.hasQueuePtr()) {
444 Register QueuePtrReg = Info.addQueuePtr(TRI);
445 MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
446 CCInfo.AllocateReg(QueuePtrReg);
447 }
448
449 if (Info.hasKernargSegmentPtr()) {
450 MachineRegisterInfo &MRI = MF.getRegInfo();
451 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
452 const LLT P4 = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
453 Register VReg = MRI.createGenericVirtualRegister(P4);
454 MRI.addLiveIn(InputPtrReg, VReg);
455 B.getMBB().addLiveIn(InputPtrReg);
456 B.buildCopy(VReg, InputPtrReg);
457 CCInfo.AllocateReg(InputPtrReg);
458 }
459
460 if (Info.hasDispatchID()) {
461 Register DispatchIDReg = Info.addDispatchID(TRI);
462 MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
463 CCInfo.AllocateReg(DispatchIDReg);
464 }
465
466 if (Info.hasFlatScratchInit()) {
467 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
468 MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
469 CCInfo.AllocateReg(FlatScratchInitReg);
470 }
471
472 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
473 // these from the dispatch pointer.
474}
475
476bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
477 MachineIRBuilder &B, const Function &F,
478 ArrayRef<ArrayRef<Register>> VRegs) const {
479 MachineFunction &MF = B.getMF();
480 const GCNSubtarget *Subtarget = &MF.getSubtarget<GCNSubtarget>();
481 MachineRegisterInfo &MRI = MF.getRegInfo();
482 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
483 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
484 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
485 const DataLayout &DL = F.getParent()->getDataLayout();
486
487 Info->allocateModuleLDSGlobal(F.getParent());
488
489 SmallVector<CCValAssign, 16> ArgLocs;
490 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
491
492 allocateHSAUserSGPRs(CCInfo, B, MF, *TRI, *Info);
493
494 unsigned i = 0;
495 const Align KernArgBaseAlign(16);
496 const unsigned BaseOffset = Subtarget->getExplicitKernelArgOffset(F);
497 uint64_t ExplicitArgOffset = 0;
498
499 // TODO: Align down to dword alignment and extract bits for extending loads.
500 for (auto &Arg : F.args()) {
501 const bool IsByRef = Arg.hasByRefAttr();
502 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
503 unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
504 if (AllocSize == 0)
505 continue;
506
507 MaybeAlign ABIAlign = IsByRef ? Arg.getParamAlign() : None;
508 if (!ABIAlign)
509 ABIAlign = DL.getABITypeAlign(ArgTy);
510
511 uint64_t ArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;
512 ExplicitArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;
513
514 if (Arg.use_empty()) {
515 ++i;
516 continue;
517 }
518
519 Align Alignment = commonAlignment(KernArgBaseAlign, ArgOffset);
520
521 if (IsByRef) {
522 unsigned ByRefAS = cast<PointerType>(Arg.getType())->getAddressSpace();
523
524 assert(VRegs[i].size() == 1 &&(static_cast <bool> (VRegs[i].size() == 1 && "expected only one register for byval pointers"
) ? void (0) : __assert_fail ("VRegs[i].size() == 1 && \"expected only one register for byval pointers\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 525, __extension__ __PRETTY_FUNCTION__))
525 "expected only one register for byval pointers")(static_cast <bool> (VRegs[i].size() == 1 && "expected only one register for byval pointers"
) ? void (0) : __assert_fail ("VRegs[i].size() == 1 && \"expected only one register for byval pointers\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 525, __extension__ __PRETTY_FUNCTION__))
;
526 if (ByRefAS == AMDGPUAS::CONSTANT_ADDRESS) {
527 lowerParameterPtr(VRegs[i][0], B, ArgTy, ArgOffset);
528 } else {
529 const LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
530 Register PtrReg = MRI.createGenericVirtualRegister(ConstPtrTy);
531 lowerParameterPtr(PtrReg, B, ArgTy, ArgOffset);
532
533 B.buildAddrSpaceCast(VRegs[i][0], PtrReg);
534 }
535 } else {
536 ArrayRef<Register> OrigArgRegs = VRegs[i];
537 Register ArgReg =
538 OrigArgRegs.size() == 1
539 ? OrigArgRegs[0]
540 : MRI.createGenericVirtualRegister(getLLTForType(*ArgTy, DL));
541
542 lowerParameter(B, ArgTy, ArgOffset, Alignment, ArgReg);
543 if (OrigArgRegs.size() > 1)
544 unpackRegs(OrigArgRegs, ArgReg, ArgTy, B);
545 }
546
547 ++i;
548 }
549
550 TLI.allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
551 TLI.allocateSystemSGPRs(CCInfo, MF, *Info, F.getCallingConv(), false);
552 return true;
553}
554
555bool AMDGPUCallLowering::lowerFormalArguments(
556 MachineIRBuilder &B, const Function &F, ArrayRef<ArrayRef<Register>> VRegs,
557 FunctionLoweringInfo &FLI) const {
558 CallingConv::ID CC = F.getCallingConv();
559
560 // The infrastructure for normal calling convention lowering is essentially
561 // useless for kernels. We want to avoid any kind of legalization or argument
562 // splitting.
563 if (CC == CallingConv::AMDGPU_KERNEL)
564 return lowerFormalArgumentsKernel(B, F, VRegs);
565
566 const bool IsGraphics = AMDGPU::isGraphics(CC);
567 const bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC);
568
569 MachineFunction &MF = B.getMF();
570 MachineBasicBlock &MBB = B.getMBB();
571 MachineRegisterInfo &MRI = MF.getRegInfo();
572 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
573 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
574 const SIRegisterInfo *TRI = Subtarget.getRegisterInfo();
575 const DataLayout &DL = F.getParent()->getDataLayout();
576
577 Info->allocateModuleLDSGlobal(F.getParent());
578
579 SmallVector<CCValAssign, 16> ArgLocs;
580 CCState CCInfo(CC, F.isVarArg(), MF, ArgLocs, F.getContext());
581
582 if (!IsEntryFunc) {
583 Register ReturnAddrReg = TRI->getReturnAddressReg(MF);
584 Register LiveInReturn = MF.addLiveIn(ReturnAddrReg,
585 &AMDGPU::SGPR_64RegClass);
586 MBB.addLiveIn(ReturnAddrReg);
587 B.buildCopy(LiveInReturn, ReturnAddrReg);
588 }
589
590 if (Info->hasImplicitBufferPtr()) {
591 Register ImplicitBufferPtrReg = Info->addImplicitBufferPtr(*TRI);
592 MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
593 CCInfo.AllocateReg(ImplicitBufferPtrReg);
594 }
595
596 SmallVector<ArgInfo, 32> SplitArgs;
597 unsigned Idx = 0;
598 unsigned PSInputNum = 0;
599
600 // Insert the hidden sret parameter if the return value won't fit in the
601 // return registers.
602 if (!FLI.CanLowerReturn)
603 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL);
604
605 for (auto &Arg : F.args()) {
606 if (DL.getTypeStoreSize(Arg.getType()) == 0)
607 continue;
608
609 const bool InReg = Arg.hasAttribute(Attribute::InReg);
610
611 // SGPR arguments to functions not implemented.
612 if (!IsGraphics && InReg)
613 return false;
614
615 if (Arg.hasAttribute(Attribute::SwiftSelf) ||
616 Arg.hasAttribute(Attribute::SwiftError) ||
617 Arg.hasAttribute(Attribute::Nest))
618 return false;
619
620 if (CC == CallingConv::AMDGPU_PS && !InReg && PSInputNum <= 15) {
621 const bool ArgUsed = !Arg.use_empty();
622 bool SkipArg = !ArgUsed && !Info->isPSInputAllocated(PSInputNum);
623
624 if (!SkipArg) {
625 Info->markPSInputAllocated(PSInputNum);
626 if (ArgUsed)
627 Info->markPSInputEnabled(PSInputNum);
628 }
629
630 ++PSInputNum;
631
632 if (SkipArg) {
633 for (int I = 0, E = VRegs[Idx].size(); I != E; ++I)
634 B.buildUndef(VRegs[Idx][I]);
635
636 ++Idx;
637 continue;
638 }
639 }
640
641 ArgInfo OrigArg(VRegs[Idx], Arg);
642 const unsigned OrigArgIdx = Idx + AttributeList::FirstArgIndex;
643 setArgFlags(OrigArg, OrigArgIdx, DL, F);
644
645 splitToValueTypes(OrigArg, SplitArgs, DL, CC);
646 ++Idx;
647 }
648
649 // At least one interpolation mode must be enabled or else the GPU will
650 // hang.
651 //
652 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
653 // set PSInputAddr, the user wants to enable some bits after the compilation
654 // based on run-time states. Since we can't know what the final PSInputEna
655 // will look like, so we shouldn't do anything here and the user should take
656 // responsibility for the correct programming.
657 //
658 // Otherwise, the following restrictions apply:
659 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
660 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
661 // enabled too.
662 if (CC == CallingConv::AMDGPU_PS) {
663 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
664 ((Info->getPSInputAddr() & 0xF) == 0 &&
665 Info->isPSInputAllocated(11))) {
666 CCInfo.AllocateReg(AMDGPU::VGPR0);
667 CCInfo.AllocateReg(AMDGPU::VGPR1);
668 Info->markPSInputAllocated(0);
669 Info->markPSInputEnabled(0);
670 }
671
672 if (Subtarget.isAmdPalOS()) {
673 // For isAmdPalOS, the user does not enable some bits after compilation
674 // based on run-time states; the register values being generated here are
675 // the final ones set in hardware. Therefore we need to apply the
676 // workaround to PSInputAddr and PSInputEnable together. (The case where
677 // a bit is set in PSInputAddr but not PSInputEnable is where the frontend
678 // set up an input arg for a particular interpolation mode, but nothing
679 // uses that input arg. Really we should have an earlier pass that removes
680 // such an arg.)
681 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
682 if ((PsInputBits & 0x7F) == 0 ||
683 ((PsInputBits & 0xF) == 0 &&
684 (PsInputBits >> 11 & 1)))
685 Info->markPSInputEnabled(
686 countTrailingZeros(Info->getPSInputAddr(), ZB_Undefined));
687 }
688 }
689
690 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
691 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CC, F.isVarArg());
692
693 if (!MBB.empty())
694 B.setInstr(*MBB.begin());
695
696 if (!IsEntryFunc) {
697 // For the fixed ABI, pass workitem IDs in the last argument register.
698 if (AMDGPUTargetMachine::EnableFixedFunctionABI)
699 TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
700 }
701
702 IncomingValueAssigner Assigner(AssignFn);
703 if (!determineAssignments(Assigner, SplitArgs, CCInfo))
704 return false;
705
706 FormalArgHandler Handler(B, MRI);
707 if (!handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, B))
708 return false;
709
710 uint64_t StackOffset = Assigner.StackOffset;
711
712 if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) {
713 // Special inputs come after user arguments.
714 TLI.allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
715 }
716
717 // Start adding system SGPRs.
718 if (IsEntryFunc) {
719 TLI.allocateSystemSGPRs(CCInfo, MF, *Info, CC, IsGraphics);
720 } else {
721 if (!Subtarget.enableFlatScratch())
722 CCInfo.AllocateReg(Info->getScratchRSrcReg());
723 TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
724 }
725
726 // When we tail call, we need to check if the callee's arguments will fit on
727 // the caller's stack. So, whenever we lower formal arguments, we should keep
728 // track of this information, since we might lower a tail call in this
729 // function later.
730 Info->setBytesInStackArgArea(StackOffset);
731
732 // Move back to the end of the basic block.
733 B.setMBB(MBB);
734
735 return true;
736}
737
738bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
739 CCState &CCInfo,
740 SmallVectorImpl<std::pair<MCRegister, Register>> &ArgRegs,
741 CallLoweringInfo &Info) const {
742 MachineFunction &MF = MIRBuilder.getMF();
743
744 const AMDGPUFunctionArgInfo *CalleeArgInfo
745 = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
746
747 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
748 const AMDGPUFunctionArgInfo &CallerArgInfo = MFI->getArgInfo();
749
750
751 // TODO: Unify with private memory register handling. This is complicated by
752 // the fact that at least in kernels, the input argument is not necessarily
753 // in the same location as the input.
754 AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = {
755 AMDGPUFunctionArgInfo::DISPATCH_PTR,
756 AMDGPUFunctionArgInfo::QUEUE_PTR,
757 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR,
758 AMDGPUFunctionArgInfo::DISPATCH_ID,
759 AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
760 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
761 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z
762 };
763
764 MachineRegisterInfo &MRI = MF.getRegInfo();
765
766 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
767 const AMDGPULegalizerInfo *LI
768 = static_cast<const AMDGPULegalizerInfo*>(ST.getLegalizerInfo());
769
770 for (auto InputID : InputRegs) {
771 const ArgDescriptor *OutgoingArg;
772 const TargetRegisterClass *ArgRC;
773 LLT ArgTy;
774
775 std::tie(OutgoingArg, ArgRC, ArgTy) =
776 CalleeArgInfo->getPreloadedValue(InputID);
777 if (!OutgoingArg)
778 continue;
779
780 const ArgDescriptor *IncomingArg;
781 const TargetRegisterClass *IncomingArgRC;
782 std::tie(IncomingArg, IncomingArgRC, ArgTy) =
783 CallerArgInfo.getPreloadedValue(InputID);
784 assert(IncomingArgRC == ArgRC)(static_cast <bool> (IncomingArgRC == ArgRC) ? void (0)
: __assert_fail ("IncomingArgRC == ArgRC", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 784, __extension__ __PRETTY_FUNCTION__))
;
785
786 Register InputReg = MRI.createGenericVirtualRegister(ArgTy);
787
788 if (IncomingArg) {
789 LI->loadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);
790 } else {
791 assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR)(static_cast <bool> (InputID == AMDGPUFunctionArgInfo::
IMPLICIT_ARG_PTR) ? void (0) : __assert_fail ("InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 791, __extension__ __PRETTY_FUNCTION__))
;
792 LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder);
793 }
794
795 if (OutgoingArg->isRegister()) {
796 ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);
797 if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
798 report_fatal_error("failed to allocate implicit input argument");
799 } else {
800 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Unhandled stack passed implicit input argument\n"
; } } while (false)
;
801 return false;
802 }
803 }
804
805 // Pack workitem IDs into a single register or pass it as is if already
806 // packed.
807 const ArgDescriptor *OutgoingArg;
808 const TargetRegisterClass *ArgRC;
809 LLT ArgTy;
810
811 std::tie(OutgoingArg, ArgRC, ArgTy) =
812 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
813 if (!OutgoingArg)
814 std::tie(OutgoingArg, ArgRC, ArgTy) =
815 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
816 if (!OutgoingArg)
817 std::tie(OutgoingArg, ArgRC, ArgTy) =
818 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
819 if (!OutgoingArg)
820 return false;
821
822 auto WorkitemIDX =
823 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
824 auto WorkitemIDY =
825 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
826 auto WorkitemIDZ =
827 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
828
829 const ArgDescriptor *IncomingArgX = std::get<0>(WorkitemIDX);
830 const ArgDescriptor *IncomingArgY = std::get<0>(WorkitemIDY);
831 const ArgDescriptor *IncomingArgZ = std::get<0>(WorkitemIDZ);
832 const LLT S32 = LLT::scalar(32);
833
834 // If incoming ids are not packed we need to pack them.
835 // FIXME: Should consider known workgroup size to eliminate known 0 cases.
836 Register InputReg;
837 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX) {
838 InputReg = MRI.createGenericVirtualRegister(S32);
839 LI->loadInputValue(InputReg, MIRBuilder, IncomingArgX,
840 std::get<1>(WorkitemIDX), std::get<2>(WorkitemIDX));
841 }
842
843 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY) {
844 Register Y = MRI.createGenericVirtualRegister(S32);
845 LI->loadInputValue(Y, MIRBuilder, IncomingArgY, std::get<1>(WorkitemIDY),
846 std::get<2>(WorkitemIDY));
847
848 Y = MIRBuilder.buildShl(S32, Y, MIRBuilder.buildConstant(S32, 10)).getReg(0);
849 InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Y).getReg(0) : Y;
850 }
851
852 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ) {
853 Register Z = MRI.createGenericVirtualRegister(S32);
854 LI->loadInputValue(Z, MIRBuilder, IncomingArgZ, std::get<1>(WorkitemIDZ),
855 std::get<2>(WorkitemIDZ));
856
857 Z = MIRBuilder.buildShl(S32, Z, MIRBuilder.buildConstant(S32, 20)).getReg(0);
858 InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Z).getReg(0) : Z;
859 }
860
861 if (!InputReg) {
862 InputReg = MRI.createGenericVirtualRegister(S32);
863
864 // Workitem ids are already packed, any of present incoming arguments will
865 // carry all required fields.
866 ArgDescriptor IncomingArg = ArgDescriptor::createArg(
867 IncomingArgX ? *IncomingArgX :
868 IncomingArgY ? *IncomingArgY : *IncomingArgZ, ~0u);
869 LI->loadInputValue(InputReg, MIRBuilder, &IncomingArg,
870 &AMDGPU::VGPR_32RegClass, S32);
871 }
872
873 if (OutgoingArg->isRegister()) {
874 ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);
875 if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
876 report_fatal_error("failed to allocate implicit input argument");
877 } else {
878 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Unhandled stack passed implicit input argument\n"
; } } while (false)
;
879 return false;
880 }
881
882 return true;
883}
884
885/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
886/// CC.
887static std::pair<CCAssignFn *, CCAssignFn *>
888getAssignFnsForCC(CallingConv::ID CC, const SITargetLowering &TLI) {
889 return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
890}
891
892static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
893 bool IsTailCall) {
894 return IsTailCall ? AMDGPU::SI_TCRETURN : AMDGPU::SI_CALL;
895}
896
897// Add operands to call instruction to track the callee.
898static bool addCallTargetOperands(MachineInstrBuilder &CallInst,
899 MachineIRBuilder &MIRBuilder,
900 AMDGPUCallLowering::CallLoweringInfo &Info) {
901 if (Info.Callee.isReg()) {
902 CallInst.addReg(Info.Callee.getReg());
903 CallInst.addImm(0);
904 } else if (Info.Callee.isGlobal() && Info.Callee.getOffset() == 0) {
905 // The call lowering lightly assumed we can directly encode a call target in
906 // the instruction, which is not the case. Materialize the address here.
907 const GlobalValue *GV = Info.Callee.getGlobal();
908 auto Ptr = MIRBuilder.buildGlobalValue(
909 LLT::pointer(GV->getAddressSpace(), 64), GV);
910 CallInst.addReg(Ptr.getReg(0));
911 CallInst.add(Info.Callee);
912 } else
913 return false;
914
915 return true;
916}
917
918bool AMDGPUCallLowering::doCallerAndCalleePassArgsTheSameWay(
919 CallLoweringInfo &Info, MachineFunction &MF,
920 SmallVectorImpl<ArgInfo> &InArgs) const {
921 const Function &CallerF = MF.getFunction();
922 CallingConv::ID CalleeCC = Info.CallConv;
923 CallingConv::ID CallerCC = CallerF.getCallingConv();
924
925 // If the calling conventions match, then everything must be the same.
926 if (CalleeCC == CallerCC)
927 return true;
928
929 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
930
931 // Make sure that the caller and callee preserve all of the same registers.
932 auto TRI = ST.getRegisterInfo();
933
934 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
935 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
936 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
937 return false;
938
939 // Check if the caller and callee will handle arguments in the same way.
940 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
941 CCAssignFn *CalleeAssignFnFixed;
942 CCAssignFn *CalleeAssignFnVarArg;
943 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
944 getAssignFnsForCC(CalleeCC, TLI);
945
946 CCAssignFn *CallerAssignFnFixed;
947 CCAssignFn *CallerAssignFnVarArg;
948 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
949 getAssignFnsForCC(CallerCC, TLI);
950
951 // FIXME: We are not accounting for potential differences in implicitly passed
952 // inputs, but only the fixed ABI is supported now anyway.
953 IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
954 CalleeAssignFnVarArg);
955 IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
956 CallerAssignFnVarArg);
957 return resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner);
958}
959
960bool AMDGPUCallLowering::areCalleeOutgoingArgsTailCallable(
961 CallLoweringInfo &Info, MachineFunction &MF,
962 SmallVectorImpl<ArgInfo> &OutArgs) const {
963 // If there are no outgoing arguments, then we are done.
964 if (OutArgs.empty())
965 return true;
966
967 const Function &CallerF = MF.getFunction();
968 CallingConv::ID CalleeCC = Info.CallConv;
969 CallingConv::ID CallerCC = CallerF.getCallingConv();
970 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
971
972 CCAssignFn *AssignFnFixed;
973 CCAssignFn *AssignFnVarArg;
974 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
975
976 // We have outgoing arguments. Make sure that we can tail call with them.
977 SmallVector<CCValAssign, 16> OutLocs;
978 CCState OutInfo(CalleeCC, false, MF, OutLocs, CallerF.getContext());
979 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
980
981 if (!determineAssignments(Assigner, OutArgs, OutInfo)) {
982 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Could not analyze call operands.\n"
; } } while (false)
;
983 return false;
984 }
985
986 // Make sure that they can fit on the caller's stack.
987 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
988 if (OutInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) {
989 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Cannot fit call operands on caller's stack.\n"
; } } while (false)
;
990 return false;
991 }
992
993 // Verify that the parameters in callee-saved registers match.
994 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
995 const SIRegisterInfo *TRI = ST.getRegisterInfo();
996 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
997 MachineRegisterInfo &MRI = MF.getRegInfo();
998 return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);
999}
1000
1001/// Return true if the calling convention is one that we can guarantee TCO for.
1002static bool canGuaranteeTCO(CallingConv::ID CC) {
1003 return CC == CallingConv::Fast;
1004}
1005
1006/// Return true if we might ever do TCO for calls with this calling convention.
1007static bool mayTailCallThisCC(CallingConv::ID CC) {
1008 switch (CC) {
1009 case CallingConv::C:
1010 case CallingConv::AMDGPU_Gfx:
1011 return true;
1012 default:
1013 return canGuaranteeTCO(CC);
1014 }
1015}
1016
1017bool AMDGPUCallLowering::isEligibleForTailCallOptimization(
1018 MachineIRBuilder &B, CallLoweringInfo &Info,
1019 SmallVectorImpl<ArgInfo> &InArgs, SmallVectorImpl<ArgInfo> &OutArgs) const {
1020 // Must pass all target-independent checks in order to tail call optimize.
1021 if (!Info.IsTailCall)
1022 return false;
1023
1024 MachineFunction &MF = B.getMF();
1025 const Function &CallerF = MF.getFunction();
1026 CallingConv::ID CalleeCC = Info.CallConv;
1027 CallingConv::ID CallerCC = CallerF.getCallingConv();
1028
1029 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1030 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
1031 // Kernels aren't callable, and don't have a live in return address so it
1032 // doesn't make sense to do a tail call with entry functions.
1033 if (!CallerPreserved)
1034 return false;
1035
1036 if (!mayTailCallThisCC(CalleeCC)) {
1037 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Calling convention cannot be tail called.\n"
; } } while (false)
;
1038 return false;
1039 }
1040
1041 if (any_of(CallerF.args(), [](const Argument &A) {
1042 return A.hasByValAttr() || A.hasSwiftErrorAttr();
1043 })) {
1044 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Cannot tail call from callers with byval "
"or swifterror arguments\n"; } } while (false)
1045 "or swifterror arguments\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Cannot tail call from callers with byval "
"or swifterror arguments\n"; } } while (false)
;
1046 return false;
1047 }
1048
1049 // If we have -tailcallopt, then we're done.
1050 if (MF.getTarget().Options.GuaranteedTailCallOpt)
1051 return canGuaranteeTCO(CalleeCC) && CalleeCC == CallerF.getCallingConv();
1052
1053 // Verify that the incoming and outgoing arguments from the callee are
1054 // safe to tail call.
1055 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
1056 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
1057 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
1058 << "... Caller and callee have incompatible calling conventions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
;
1059 return false;
1060 }
1061
1062 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
1063 return false;
1064
1065 LLVM_DEBUG(dbgs() << "... Call is eligible for tail call optimization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Call is eligible for tail call optimization.\n"
; } } while (false)
;
1066 return true;
1067}
1068
1069// Insert outgoing implicit arguments for a call, by inserting copies to the
1070// implicit argument registers and adding the necessary implicit uses to the
1071// call instruction.
1072void AMDGPUCallLowering::handleImplicitCallArguments(
1073 MachineIRBuilder &MIRBuilder, MachineInstrBuilder &CallInst,
1074 const GCNSubtarget &ST, const SIMachineFunctionInfo &FuncInfo,
1075 ArrayRef<std::pair<MCRegister, Register>> ImplicitArgRegs) const {
1076 if (!ST.enableFlatScratch()) {
1077 // Insert copies for the SRD. In the HSA case, this should be an identity
1078 // copy.
1079 auto ScratchRSrcReg =
1080 MIRBuilder.buildCopy(LLT::vector(4, 32), FuncInfo.getScratchRSrcReg());
1081 MIRBuilder.buildCopy(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
1082 CallInst.addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Implicit);
1083 }
1084
1085 for (std::pair<MCRegister, Register> ArgReg : ImplicitArgRegs) {
1086 MIRBuilder.buildCopy((Register)ArgReg.first, ArgReg.second);
1087 CallInst.addReg(ArgReg.first, RegState::Implicit);
1088 }
1089}
1090
1091bool AMDGPUCallLowering::lowerTailCall(
1092 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1093 SmallVectorImpl<ArgInfo> &OutArgs) const {
1094 MachineFunction &MF = MIRBuilder.getMF();
1095 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1096 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1097 const Function &F = MF.getFunction();
1098 MachineRegisterInfo &MRI = MF.getRegInfo();
1099 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1100
1101 // True when we're tail calling, but without -tailcallopt.
1102 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt;
1103
1104 // Find out which ABI gets to decide where things go.
1105 CallingConv::ID CalleeCC = Info.CallConv;
1106 CCAssignFn *AssignFnFixed;
1107 CCAssignFn *AssignFnVarArg;
1108 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
1109
1110 MachineInstrBuilder CallSeqStart;
1111 if (!IsSibCall)
1112 CallSeqStart = MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP);
1113
1114 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true);
1115 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1116 if (!addCallTargetOperands(MIB, MIRBuilder, Info))
1117 return false;
1118
1119 // Byte offset for the tail call. When we are sibcalling, this will always
1120 // be 0.
1121 MIB.addImm(0);
1122
1123 // Tell the call which registers are clobbered.
1124 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1125 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1126 MIB.addRegMask(Mask);
1127
1128 // FPDiff is the byte offset of the call's argument area from the callee's.
1129 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1130 // by this amount for a tail call. In a sibling call it must be 0 because the
1131 // caller will deallocate the entire stack and the callee still expects its
1132 // arguments to begin at SP+0.
1133 int FPDiff = 0;
1134
1135 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1136 // by -tailcallopt. For sibcalls, the memory operands for the call are
1137 // already available in the caller's incoming argument space.
1138 unsigned NumBytes = 0;
1139 if (!IsSibCall) {
1140 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1141 // before handling assignments, because FPDiff must be known for memory
1142 // arguments.
1143 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1144 SmallVector<CCValAssign, 16> OutLocs;
1145 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1146
1147 // FIXME: Not accounting for callee implicit inputs
1148 OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg);
1149 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo))
1150 return false;
1151
1152 // The callee will pop the argument stack as a tail call. Thus, we must
1153 // keep it 16-byte aligned.
1154 NumBytes = alignTo(OutInfo.getNextStackOffset(), ST.getStackAlignment());
1155
1156 // FPDiff will be negative if this tail call requires more space than we
1157 // would automatically have in our incoming argument space. Positive if we
1158 // actually shrink the stack.
1159 FPDiff = NumReusableBytes - NumBytes;
1160
1161 // The stack pointer must be 16-byte aligned at all times it's used for a
1162 // memory operation, which in practice means at *all* times and in
1163 // particular across call boundaries. Therefore our own arguments started at
1164 // a 16-byte aligned SP and the delta applied for the tail call should
1165 // satisfy the same constraint.
1166 assert(isAligned(ST.getStackAlignment(), FPDiff) &&(static_cast <bool> (isAligned(ST.getStackAlignment(), FPDiff
) && "unaligned stack on tail call") ? void (0) : __assert_fail
("isAligned(ST.getStackAlignment(), FPDiff) && \"unaligned stack on tail call\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 1167, __extension__ __PRETTY_FUNCTION__))
1167 "unaligned stack on tail call")(static_cast <bool> (isAligned(ST.getStackAlignment(), FPDiff
) && "unaligned stack on tail call") ? void (0) : __assert_fail
("isAligned(ST.getStackAlignment(), FPDiff) && \"unaligned stack on tail call\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 1167, __extension__ __PRETTY_FUNCTION__))
;
1168 }
1169
1170 SmallVector<CCValAssign, 16> ArgLocs;
1171 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());
1172
1173 // We could pass MIB and directly add the implicit uses to the call
1174 // now. However, as an aesthetic choice, place implicit argument operands
1175 // after the ordinary user argument registers.
1176 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
1177
1178 if (AMDGPUTargetMachine::EnableFixedFunctionABI &&
1179 Info.CallConv != CallingConv::AMDGPU_Gfx) {
1180 // With a fixed ABI, allocate fixed registers before user arguments.
1181 if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))
1182 return false;
1183 }
1184
1185 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1186
1187 if (!determineAssignments(Assigner, OutArgs, CCInfo))
1188 return false;
1189
1190 // Do the actual argument marshalling.
1191 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, true, FPDiff);
1192 if (!handleAssignments(Handler, OutArgs, CCInfo, ArgLocs, MIRBuilder))
1193 return false;
1194
1195 handleImplicitCallArguments(MIRBuilder, MIB, ST, *FuncInfo, ImplicitArgRegs);
1196
1197 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1198 // sequence start and end here.
1199 if (!IsSibCall) {
1200 MIB->getOperand(1).setImm(FPDiff);
1201 CallSeqStart.addImm(NumBytes).addImm(0);
1202 // End the call sequence *before* emitting the call. Normally, we would
1203 // tidy the frame up after the call. However, here, we've laid out the
1204 // parameters so that when SP is reset, they will be in the correct
1205 // location.
1206 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKDOWN).addImm(NumBytes).addImm(0);
1207 }
1208
1209 // Now we can add the actual call instruction to the correct basic block.
1210 MIRBuilder.insertInstr(MIB);
1211
1212 // If Callee is a reg, since it is used by a target specific
1213 // instruction, it must have a register class matching the
1214 // constraint of that instruction.
1215
1216 // FIXME: We should define regbankselectable call instructions to handle
1217 // divergent call targets.
1218 if (MIB->getOperand(0).isReg()) {
1219 MIB->getOperand(0).setReg(constrainOperandRegClass(
1220 MF, *TRI, MRI, *ST.getInstrInfo(), *ST.getRegBankInfo(), *MIB,
1221 MIB->getDesc(), MIB->getOperand(0), 0));
1222 }
1223
1224 MF.getFrameInfo().setHasTailCall();
1225 Info.LoweredTailCall = true;
1226 return true;
1227}
1228
1229bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
1230 CallLoweringInfo &Info) const {
1231 if (Info.IsVarArg) {
1232 LLVM_DEBUG(dbgs() << "Variadic functions not implemented\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Variadic functions not implemented\n"
; } } while (false)
;
1233 return false;
1234 }
1235
1236 MachineFunction &MF = MIRBuilder.getMF();
1237 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1238 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1239
1240 const Function &F = MF.getFunction();
1241 MachineRegisterInfo &MRI = MF.getRegInfo();
1242 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1243 const DataLayout &DL = F.getParent()->getDataLayout();
1244
1245 if (!AMDGPUTargetMachine::EnableFixedFunctionABI &&
1246 Info.CallConv != CallingConv::AMDGPU_Gfx) {
1247 LLVM_DEBUG(dbgs() << "Variable function ABI not implemented\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Variable function ABI not implemented\n"
; } } while (false)
;
1248 return false;
1249 }
1250
1251 SmallVector<ArgInfo, 8> OutArgs;
1252 for (auto &OrigArg : Info.OrigArgs)
1253 splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);
1254
1255 SmallVector<ArgInfo, 8> InArgs;
1256 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy())
1257 splitToValueTypes(Info.OrigRet, InArgs, DL, Info.CallConv);
1258
1259 // If we can lower as a tail call, do that instead.
1260 bool CanTailCallOpt =
1261 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
1262
1263 // We must emit a tail call if we have musttail.
1264 if (Info.IsMustTailCall && !CanTailCallOpt) {
1265 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Failed to lower musttail call as tail call\n"
; } } while (false)
;
1266 return false;
1267 }
1268
1269 if (CanTailCallOpt)
1270 return lowerTailCall(MIRBuilder, Info, OutArgs);
1271
1272 // Find out which ABI gets to decide where things go.
1273 CCAssignFn *AssignFnFixed;
1274 CCAssignFn *AssignFnVarArg;
1275 std::tie(AssignFnFixed, AssignFnVarArg) =
1276 getAssignFnsForCC(Info.CallConv, TLI);
1277
1278 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP)
1279 .addImm(0)
1280 .addImm(0);
1281
1282 // Create a temporarily-floating call instruction so we can add the implicit
1283 // uses of arg registers.
1284 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
1285
1286 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1287 MIB.addDef(TRI->getReturnAddressReg(MF));
1288
1289 if (!addCallTargetOperands(MIB, MIRBuilder, Info))
1290 return false;
1291
1292 // Tell the call which registers are clobbered.
1293 const uint32_t *Mask = TRI->getCallPreservedMask(MF, Info.CallConv);
1294 MIB.addRegMask(Mask);
1295
1296 SmallVector<CCValAssign, 16> ArgLocs;
1297 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());
1298
1299 // We could pass MIB and directly add the implicit uses to the call
1300 // now. However, as an aesthetic choice, place implicit argument operands
1301 // after the ordinary user argument registers.
1302 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
1303
1304 if (AMDGPUTargetMachine::EnableFixedFunctionABI &&
1305 Info.CallConv != CallingConv::AMDGPU_Gfx) {
1306 // With a fixed ABI, allocate fixed registers before user arguments.
1307 if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))
1308 return false;
1309 }
1310
1311 // Do the actual argument marshalling.
1312 SmallVector<Register, 8> PhysRegs;
1313
1314 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1315 if (!determineAssignments(Assigner, OutArgs, CCInfo))
1316 return false;
1317
1318 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, false);
1319 if (!handleAssignments(Handler, OutArgs, CCInfo, ArgLocs, MIRBuilder))
1320 return false;
1321
1322 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1323
1324 handleImplicitCallArguments(MIRBuilder, MIB, ST, *MFI, ImplicitArgRegs);
1325
1326 // Get a count of how many bytes are to be pushed on the stack.
1327 unsigned NumBytes = CCInfo.getNextStackOffset();
1328
1329 // If Callee is a reg, since it is used by a target specific
1330 // instruction, it must have a register class matching the
1331 // constraint of that instruction.
1332
1333 // FIXME: We should define regbankselectable call instructions to handle
1334 // divergent call targets.
1335 if (MIB->getOperand(1).isReg()) {
1336 MIB->getOperand(1).setReg(constrainOperandRegClass(
1337 MF, *TRI, MRI, *ST.getInstrInfo(),
1338 *ST.getRegBankInfo(), *MIB, MIB->getDesc(), MIB->getOperand(1),
1339 1));
1340 }
1341
1342 // Now we can add the actual call instruction to the correct position.
1343 MIRBuilder.insertInstr(MIB);
1344
1345 // Finally we can copy the returned value back into its virtual-register. In
1346 // symmetry with the arguments, the physical register must be an
1347 // implicit-define of the call instruction.
1348 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1349 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv,
1350 Info.IsVarArg);
1351 IncomingValueAssigner Assigner(RetAssignFn);
1352 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1353 if (!determineAndHandleAssignments(Handler, Assigner, InArgs, MIRBuilder,
1354 Info.CallConv, Info.IsVarArg))
1355 return false;
1356 }
1357
1358 uint64_t CalleePopBytes = NumBytes;
1359
1360 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKDOWN)
1361 .addImm(0)
1362 .addImm(CalleePopBytes);
1363
1364 if (!Info.CanLowerReturn) {
1365 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,
1366 Info.DemoteRegister, Info.DemoteStackIndex);
1367 }
1368
1369 return true;
1370}