Bug Summary

File:build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
Warning:line 373, column 25
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name AMDGPUCallLowering.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/AMDGPU -I include -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-10-03-140002-15933-1 -x c++ /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
1//===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUCallLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPULegalizerInfo.h"
18#include "AMDGPUTargetMachine.h"
19#include "SIMachineFunctionInfo.h"
20#include "SIRegisterInfo.h"
21#include "llvm/CodeGen/Analysis.h"
22#include "llvm/CodeGen/FunctionLoweringInfo.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/IR/IntrinsicsAMDGPU.h"
26
27#define DEBUG_TYPE"amdgpu-call-lowering" "amdgpu-call-lowering"
28
29using namespace llvm;
30
31namespace {
32
33/// Wrapper around extendRegister to ensure we extend to a full 32-bit register.
34static Register extendRegisterMin32(CallLowering::ValueHandler &Handler,
35 Register ValVReg, CCValAssign &VA) {
36 if (VA.getLocVT().getSizeInBits() < 32) {
37 // 16-bit types are reported as legal for 32-bit registers. We need to
38 // extend and do a 32-bit copy to avoid the verifier complaining about it.
39 return Handler.MIRBuilder.buildAnyExt(LLT::scalar(32), ValVReg).getReg(0);
40 }
41
42 return Handler.extendRegister(ValVReg, VA);
43}
44
45struct AMDGPUOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
46 AMDGPUOutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
47 MachineInstrBuilder MIB)
48 : OutgoingValueHandler(B, MRI), MIB(MIB) {}
49
50 MachineInstrBuilder MIB;
51
52 Register getStackAddress(uint64_t Size, int64_t Offset,
53 MachinePointerInfo &MPO,
54 ISD::ArgFlagsTy Flags) override {
55 llvm_unreachable("not implemented")::llvm::llvm_unreachable_internal("not implemented", "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 55)
;
56 }
57
58 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
59 MachinePointerInfo &MPO, CCValAssign &VA) override {
60 llvm_unreachable("not implemented")::llvm::llvm_unreachable_internal("not implemented", "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 60)
;
61 }
62
63 void assignValueToReg(Register ValVReg, Register PhysReg,
64 CCValAssign VA) override {
65 Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);
66
67 // If this is a scalar return, insert a readfirstlane just in case the value
68 // ends up in a VGPR.
69 // FIXME: Assert this is a shader return.
70 const SIRegisterInfo *TRI
71 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
72 if (TRI->isSGPRReg(MRI, PhysReg)) {
73 auto ToSGPR = MIRBuilder.buildIntrinsic(Intrinsic::amdgcn_readfirstlane,
74 {MRI.getType(ExtReg)}, false)
75 .addReg(ExtReg);
76 ExtReg = ToSGPR.getReg(0);
77 }
78
79 MIRBuilder.buildCopy(PhysReg, ExtReg);
80 MIB.addUse(PhysReg, RegState::Implicit);
81 }
82};
83
84struct AMDGPUIncomingArgHandler : public CallLowering::IncomingValueHandler {
85 uint64_t StackUsed = 0;
86
87 AMDGPUIncomingArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
88 : IncomingValueHandler(B, MRI) {}
89
90 Register getStackAddress(uint64_t Size, int64_t Offset,
91 MachinePointerInfo &MPO,
92 ISD::ArgFlagsTy Flags) override {
93 auto &MFI = MIRBuilder.getMF().getFrameInfo();
94
95 // Byval is assumed to be writable memory, but other stack passed arguments
96 // are not.
97 const bool IsImmutable = !Flags.isByVal();
98 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);
99 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
100 auto AddrReg = MIRBuilder.buildFrameIndex(
101 LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32), FI);
102 StackUsed = std::max(StackUsed, Size + Offset);
103 return AddrReg.getReg(0);
104 }
105
106 void assignValueToReg(Register ValVReg, Register PhysReg,
107 CCValAssign VA) override {
108 markPhysRegUsed(PhysReg);
109
110 if (VA.getLocVT().getSizeInBits() < 32) {
111 // 16-bit types are reported as legal for 32-bit registers. We need to do
112 // a 32-bit copy, and truncate to avoid the verifier complaining about it.
113 auto Copy = MIRBuilder.buildCopy(LLT::scalar(32), PhysReg);
114
115 // If we have signext/zeroext, it applies to the whole 32-bit register
116 // before truncation.
117 auto Extended =
118 buildExtensionHint(VA, Copy.getReg(0), LLT(VA.getLocVT()));
119 MIRBuilder.buildTrunc(ValVReg, Extended);
120 return;
121 }
122
123 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
124 }
125
126 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
127 MachinePointerInfo &MPO, CCValAssign &VA) override {
128 MachineFunction &MF = MIRBuilder.getMF();
129
130 auto MMO = MF.getMachineMemOperand(
131 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy,
132 inferAlignFromPtrInfo(MF, MPO));
133 MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
134 }
135
136 /// How the physical register gets marked varies between formal
137 /// parameters (it's a basic-block live-in), and a call instruction
138 /// (it's an implicit-def of the BL).
139 virtual void markPhysRegUsed(unsigned PhysReg) = 0;
140};
141
142struct FormalArgHandler : public AMDGPUIncomingArgHandler {
143 FormalArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
144 : AMDGPUIncomingArgHandler(B, MRI) {}
145
146 void markPhysRegUsed(unsigned PhysReg) override {
147 MIRBuilder.getMBB().addLiveIn(PhysReg);
148 }
149};
150
151struct CallReturnHandler : public AMDGPUIncomingArgHandler {
152 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
153 MachineInstrBuilder MIB)
154 : AMDGPUIncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
155
156 void markPhysRegUsed(unsigned PhysReg) override {
157 MIB.addDef(PhysReg, RegState::Implicit);
158 }
159
160 MachineInstrBuilder MIB;
161};
162
163struct AMDGPUOutgoingArgHandler : public AMDGPUOutgoingValueHandler {
164 /// For tail calls, the byte offset of the call's argument area from the
165 /// callee's. Unused elsewhere.
166 int FPDiff;
167
168 // Cache the SP register vreg if we need it more than once in this call site.
169 Register SPReg;
170
171 bool IsTailCall;
172
173 AMDGPUOutgoingArgHandler(MachineIRBuilder &MIRBuilder,
174 MachineRegisterInfo &MRI, MachineInstrBuilder MIB,
175 bool IsTailCall = false, int FPDiff = 0)
176 : AMDGPUOutgoingValueHandler(MIRBuilder, MRI, MIB), FPDiff(FPDiff),
177 IsTailCall(IsTailCall) {}
178
179 Register getStackAddress(uint64_t Size, int64_t Offset,
180 MachinePointerInfo &MPO,
181 ISD::ArgFlagsTy Flags) override {
182 MachineFunction &MF = MIRBuilder.getMF();
183 const LLT PtrTy = LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32);
184 const LLT S32 = LLT::scalar(32);
185
186 if (IsTailCall) {
187 Offset += FPDiff;
188 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
189 auto FIReg = MIRBuilder.buildFrameIndex(PtrTy, FI);
190 MPO = MachinePointerInfo::getFixedStack(MF, FI);
191 return FIReg.getReg(0);
192 }
193
194 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
195
196 if (!SPReg) {
197 const GCNSubtarget &ST = MIRBuilder.getMF().getSubtarget<GCNSubtarget>();
198 if (ST.enableFlatScratch()) {
199 // The stack is accessed unswizzled, so we can use a regular copy.
200 SPReg = MIRBuilder.buildCopy(PtrTy,
201 MFI->getStackPtrOffsetReg()).getReg(0);
202 } else {
203 // The address we produce here, without knowing the use context, is going
204 // to be interpreted as a vector address, so we need to convert to a
205 // swizzled address.
206 SPReg = MIRBuilder.buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {PtrTy},
207 {MFI->getStackPtrOffsetReg()}).getReg(0);
208 }
209 }
210
211 auto OffsetReg = MIRBuilder.buildConstant(S32, Offset);
212
213 auto AddrReg = MIRBuilder.buildPtrAdd(PtrTy, SPReg, OffsetReg);
214 MPO = MachinePointerInfo::getStack(MF, Offset);
215 return AddrReg.getReg(0);
216 }
217
218 void assignValueToReg(Register ValVReg, Register PhysReg,
219 CCValAssign VA) override {
220 MIB.addUse(PhysReg, RegState::Implicit);
221 Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);
222 MIRBuilder.buildCopy(PhysReg, ExtReg);
223 }
224
225 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
226 MachinePointerInfo &MPO, CCValAssign &VA) override {
227 MachineFunction &MF = MIRBuilder.getMF();
228 uint64_t LocMemOffset = VA.getLocMemOffset();
229 const auto &ST = MF.getSubtarget<GCNSubtarget>();
230
231 auto MMO = MF.getMachineMemOperand(
232 MPO, MachineMemOperand::MOStore, MemTy,
233 commonAlignment(ST.getStackAlignment(), LocMemOffset));
234 MIRBuilder.buildStore(ValVReg, Addr, *MMO);
235 }
236
237 void assignValueToAddress(const CallLowering::ArgInfo &Arg,
238 unsigned ValRegIndex, Register Addr, LLT MemTy,
239 MachinePointerInfo &MPO, CCValAssign &VA) override {
240 Register ValVReg = VA.getLocInfo() != CCValAssign::LocInfo::FPExt
241 ? extendRegister(Arg.Regs[ValRegIndex], VA)
242 : Arg.Regs[ValRegIndex];
243 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
244 }
245};
246}
247
248AMDGPUCallLowering::AMDGPUCallLowering(const AMDGPUTargetLowering &TLI)
249 : CallLowering(&TLI) {
250}
251
252// FIXME: Compatibility shim
253static ISD::NodeType extOpcodeToISDExtOpcode(unsigned MIOpc) {
254 switch (MIOpc) {
255 case TargetOpcode::G_SEXT:
256 return ISD::SIGN_EXTEND;
257 case TargetOpcode::G_ZEXT:
258 return ISD::ZERO_EXTEND;
259 case TargetOpcode::G_ANYEXT:
260 return ISD::ANY_EXTEND;
261 default:
262 llvm_unreachable("not an extend opcode")::llvm::llvm_unreachable_internal("not an extend opcode", "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 262)
;
263 }
264}
265
266bool AMDGPUCallLowering::canLowerReturn(MachineFunction &MF,
267 CallingConv::ID CallConv,
268 SmallVectorImpl<BaseArgInfo> &Outs,
269 bool IsVarArg) const {
270 // For shaders. Vector types should be explicitly handled by CC.
271 if (AMDGPU::isEntryFunctionCC(CallConv))
272 return true;
273
274 SmallVector<CCValAssign, 16> ArgLocs;
275 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
276 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
277 MF.getFunction().getContext());
278
279 return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv, IsVarArg));
280}
281
282/// Lower the return value for the already existing \p Ret. This assumes that
283/// \p B's insertion point is correct.
284bool AMDGPUCallLowering::lowerReturnVal(MachineIRBuilder &B,
285 const Value *Val, ArrayRef<Register> VRegs,
286 MachineInstrBuilder &Ret) const {
287 if (!Val)
288 return true;
289
290 auto &MF = B.getMF();
291 const auto &F = MF.getFunction();
292 const DataLayout &DL = MF.getDataLayout();
293 MachineRegisterInfo *MRI = B.getMRI();
294 LLVMContext &Ctx = F.getContext();
295
296 CallingConv::ID CC = F.getCallingConv();
297 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
298
299 SmallVector<EVT, 8> SplitEVTs;
300 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
301 assert(VRegs.size() == SplitEVTs.size() &&(static_cast <bool> (VRegs.size() == SplitEVTs.size() &&
"For each split Type there should be exactly one VReg.") ? void
(0) : __assert_fail ("VRegs.size() == SplitEVTs.size() && \"For each split Type there should be exactly one VReg.\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 302, __extension__
__PRETTY_FUNCTION__))
302 "For each split Type there should be exactly one VReg.")(static_cast <bool> (VRegs.size() == SplitEVTs.size() &&
"For each split Type there should be exactly one VReg.") ? void
(0) : __assert_fail ("VRegs.size() == SplitEVTs.size() && \"For each split Type there should be exactly one VReg.\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 302, __extension__
__PRETTY_FUNCTION__))
;
303
304 SmallVector<ArgInfo, 8> SplitRetInfos;
305
306 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
307 EVT VT = SplitEVTs[i];
308 Register Reg = VRegs[i];
309 ArgInfo RetInfo(Reg, VT.getTypeForEVT(Ctx), 0);
310 setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);
311
312 if (VT.isScalarInteger()) {
313 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
314 if (RetInfo.Flags[0].isSExt()) {
315 assert(RetInfo.Regs.size() == 1 && "expect only simple return values")(static_cast <bool> (RetInfo.Regs.size() == 1 &&
"expect only simple return values") ? void (0) : __assert_fail
("RetInfo.Regs.size() == 1 && \"expect only simple return values\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 315, __extension__
__PRETTY_FUNCTION__))
;
316 ExtendOp = TargetOpcode::G_SEXT;
317 } else if (RetInfo.Flags[0].isZExt()) {
318 assert(RetInfo.Regs.size() == 1 && "expect only simple return values")(static_cast <bool> (RetInfo.Regs.size() == 1 &&
"expect only simple return values") ? void (0) : __assert_fail
("RetInfo.Regs.size() == 1 && \"expect only simple return values\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 318, __extension__
__PRETTY_FUNCTION__))
;
319 ExtendOp = TargetOpcode::G_ZEXT;
320 }
321
322 EVT ExtVT = TLI.getTypeForExtReturn(Ctx, VT,
323 extOpcodeToISDExtOpcode(ExtendOp));
324 if (ExtVT != VT) {
325 RetInfo.Ty = ExtVT.getTypeForEVT(Ctx);
326 LLT ExtTy = getLLTForType(*RetInfo.Ty, DL);
327 Reg = B.buildInstr(ExtendOp, {ExtTy}, {Reg}).getReg(0);
328 }
329 }
330
331 if (Reg != RetInfo.Regs[0]) {
332 RetInfo.Regs[0] = Reg;
333 // Reset the arg flags after modifying Reg.
334 setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);
335 }
336
337 splitToValueTypes(RetInfo, SplitRetInfos, DL, CC);
338 }
339
340 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC, F.isVarArg());
341
342 OutgoingValueAssigner Assigner(AssignFn);
343 AMDGPUOutgoingValueHandler RetHandler(B, *MRI, Ret);
344 return determineAndHandleAssignments(RetHandler, Assigner, SplitRetInfos, B,
345 CC, F.isVarArg());
346}
347
348bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &B, const Value *Val,
349 ArrayRef<Register> VRegs,
350 FunctionLoweringInfo &FLI) const {
351
352 MachineFunction &MF = B.getMF();
353 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
354 MFI->setIfReturnsVoid(!Val);
1
Assuming 'Val' is null
355
356 assert(!Val == VRegs.empty() && "Return value without a vreg")(static_cast <bool> (!Val == VRegs.empty() && "Return value without a vreg"
) ? void (0) : __assert_fail ("!Val == VRegs.empty() && \"Return value without a vreg\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 356, __extension__
__PRETTY_FUNCTION__))
;
2
Assuming the condition is true
3
'?' condition is true
357
358 CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
359 const bool IsShader = AMDGPU::isShader(CC);
360 const bool IsWaveEnd =
361 (IsShader && MFI->returnsVoid()) || AMDGPU::isKernel(CC);
4
Assuming 'IsShader' is false
362 if (IsWaveEnd
4.1
'IsWaveEnd' is false
) {
363 B.buildInstr(AMDGPU::S_ENDPGM)
364 .addImm(0);
365 return true;
366 }
367
368 unsigned ReturnOpc =
5
Taking false branch
369 IsShader
5.1
'IsShader' is false
? AMDGPU::SI_RETURN_TO_EPILOG : AMDGPU::SI_RETURN;
6
'?' condition is false
370 auto Ret = B.buildInstrNoInsert(ReturnOpc);
371
372 if (!FLI.CanLowerReturn)
7
Assuming field 'CanLowerReturn' is false
8
Taking true branch
373 insertSRetStores(B, Val->getType(), VRegs, FLI.DemoteRegister);
9
Called C++ object pointer is null
374 else if (!lowerReturnVal(B, Val, VRegs, Ret))
375 return false;
376
377 // TODO: Handle CalleeSavedRegsViaCopy.
378
379 B.insertInstr(Ret);
380 return true;
381}
382
383void AMDGPUCallLowering::lowerParameterPtr(Register DstReg, MachineIRBuilder &B,
384 uint64_t Offset) const {
385 MachineFunction &MF = B.getMF();
386 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
387 MachineRegisterInfo &MRI = MF.getRegInfo();
388 Register KernArgSegmentPtr =
389 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
390 Register KernArgSegmentVReg = MRI.getLiveInVirtReg(KernArgSegmentPtr);
391
392 auto OffsetReg = B.buildConstant(LLT::scalar(64), Offset);
393
394 B.buildPtrAdd(DstReg, KernArgSegmentVReg, OffsetReg);
395}
396
397void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, ArgInfo &OrigArg,
398 uint64_t Offset,
399 Align Alignment) const {
400 MachineFunction &MF = B.getMF();
401 const Function &F = MF.getFunction();
402 const DataLayout &DL = F.getParent()->getDataLayout();
403 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
404
405 LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
406
407 SmallVector<ArgInfo, 32> SplitArgs;
408 SmallVector<uint64_t> FieldOffsets;
409 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv(), &FieldOffsets);
410
411 unsigned Idx = 0;
412 for (ArgInfo &SplitArg : SplitArgs) {
413 Register PtrReg = B.getMRI()->createGenericVirtualRegister(PtrTy);
414 lowerParameterPtr(PtrReg, B, Offset + FieldOffsets[Idx]);
415
416 LLT ArgTy = getLLTForType(*SplitArg.Ty, DL);
417 if (SplitArg.Flags[0].isPointer()) {
418 // Compensate for losing pointeriness in splitValueTypes.
419 LLT PtrTy = LLT::pointer(SplitArg.Flags[0].getPointerAddrSpace(),
420 ArgTy.getScalarSizeInBits());
421 ArgTy = ArgTy.isVector() ? LLT::vector(ArgTy.getElementCount(), PtrTy)
422 : PtrTy;
423 }
424
425 MachineMemOperand *MMO = MF.getMachineMemOperand(
426 PtrInfo,
427 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
428 MachineMemOperand::MOInvariant,
429 ArgTy, commonAlignment(Alignment, FieldOffsets[Idx]));
430
431 assert(SplitArg.Regs.size() == 1)(static_cast <bool> (SplitArg.Regs.size() == 1) ? void (
0) : __assert_fail ("SplitArg.Regs.size() == 1", "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 431, __extension__ __PRETTY_FUNCTION__))
;
432
433 B.buildLoad(SplitArg.Regs[0], PtrReg, *MMO);
434 ++Idx;
435 }
436}
437
438// Allocate special inputs passed in user SGPRs.
439static void allocateHSAUserSGPRs(CCState &CCInfo,
440 MachineIRBuilder &B,
441 MachineFunction &MF,
442 const SIRegisterInfo &TRI,
443 SIMachineFunctionInfo &Info) {
444 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
445 if (Info.hasPrivateSegmentBuffer()) {
446 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
447 MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
448 CCInfo.AllocateReg(PrivateSegmentBufferReg);
449 }
450
451 if (Info.hasDispatchPtr()) {
452 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
453 MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
454 CCInfo.AllocateReg(DispatchPtrReg);
455 }
456
457 if (Info.hasQueuePtr() && AMDGPU::getAmdhsaCodeObjectVersion() < 5) {
458 Register QueuePtrReg = Info.addQueuePtr(TRI);
459 MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
460 CCInfo.AllocateReg(QueuePtrReg);
461 }
462
463 if (Info.hasKernargSegmentPtr()) {
464 MachineRegisterInfo &MRI = MF.getRegInfo();
465 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
466 const LLT P4 = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
467 Register VReg = MRI.createGenericVirtualRegister(P4);
468 MRI.addLiveIn(InputPtrReg, VReg);
469 B.getMBB().addLiveIn(InputPtrReg);
470 B.buildCopy(VReg, InputPtrReg);
471 CCInfo.AllocateReg(InputPtrReg);
472 }
473
474 if (Info.hasDispatchID()) {
475 Register DispatchIDReg = Info.addDispatchID(TRI);
476 MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
477 CCInfo.AllocateReg(DispatchIDReg);
478 }
479
480 if (Info.hasFlatScratchInit()) {
481 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
482 MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
483 CCInfo.AllocateReg(FlatScratchInitReg);
484 }
485
486 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
487 // these from the dispatch pointer.
488}
489
490bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
491 MachineIRBuilder &B, const Function &F,
492 ArrayRef<ArrayRef<Register>> VRegs) const {
493 MachineFunction &MF = B.getMF();
494 const GCNSubtarget *Subtarget = &MF.getSubtarget<GCNSubtarget>();
495 MachineRegisterInfo &MRI = MF.getRegInfo();
496 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
497 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
498 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
499 const DataLayout &DL = F.getParent()->getDataLayout();
500
501 Info->allocateKnownAddressLDSGlobal(F);
502
503 SmallVector<CCValAssign, 16> ArgLocs;
504 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
505
506 allocateHSAUserSGPRs(CCInfo, B, MF, *TRI, *Info);
507
508 unsigned i = 0;
509 const Align KernArgBaseAlign(16);
510 const unsigned BaseOffset = Subtarget->getExplicitKernelArgOffset(F);
511 uint64_t ExplicitArgOffset = 0;
512
513 // TODO: Align down to dword alignment and extract bits for extending loads.
514 for (auto &Arg : F.args()) {
515 const bool IsByRef = Arg.hasByRefAttr();
516 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
517 unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
518 if (AllocSize == 0)
519 continue;
520
521 MaybeAlign ParamAlign = IsByRef ? Arg.getParamAlign() : None;
522 Align ABIAlign = DL.getValueOrABITypeAlignment(ParamAlign, ArgTy);
523
524 uint64_t ArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;
525 ExplicitArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;
526
527 if (Arg.use_empty()) {
528 ++i;
529 continue;
530 }
531
532 Align Alignment = commonAlignment(KernArgBaseAlign, ArgOffset);
533
534 if (IsByRef) {
535 unsigned ByRefAS = cast<PointerType>(Arg.getType())->getAddressSpace();
536
537 assert(VRegs[i].size() == 1 &&(static_cast <bool> (VRegs[i].size() == 1 && "expected only one register for byval pointers"
) ? void (0) : __assert_fail ("VRegs[i].size() == 1 && \"expected only one register for byval pointers\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 538, __extension__
__PRETTY_FUNCTION__))
538 "expected only one register for byval pointers")(static_cast <bool> (VRegs[i].size() == 1 && "expected only one register for byval pointers"
) ? void (0) : __assert_fail ("VRegs[i].size() == 1 && \"expected only one register for byval pointers\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 538, __extension__
__PRETTY_FUNCTION__))
;
539 if (ByRefAS == AMDGPUAS::CONSTANT_ADDRESS) {
540 lowerParameterPtr(VRegs[i][0], B, ArgOffset);
541 } else {
542 const LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
543 Register PtrReg = MRI.createGenericVirtualRegister(ConstPtrTy);
544 lowerParameterPtr(PtrReg, B, ArgOffset);
545
546 B.buildAddrSpaceCast(VRegs[i][0], PtrReg);
547 }
548 } else {
549 ArgInfo OrigArg(VRegs[i], Arg, i);
550 const unsigned OrigArgIdx = i + AttributeList::FirstArgIndex;
551 setArgFlags(OrigArg, OrigArgIdx, DL, F);
552 lowerParameter(B, OrigArg, ArgOffset, Alignment);
553 }
554
555 ++i;
556 }
557
558 TLI.allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
559 TLI.allocateSystemSGPRs(CCInfo, MF, *Info, F.getCallingConv(), false);
560 return true;
561}
562
563bool AMDGPUCallLowering::lowerFormalArguments(
564 MachineIRBuilder &B, const Function &F, ArrayRef<ArrayRef<Register>> VRegs,
565 FunctionLoweringInfo &FLI) const {
566 CallingConv::ID CC = F.getCallingConv();
567
568 // The infrastructure for normal calling convention lowering is essentially
569 // useless for kernels. We want to avoid any kind of legalization or argument
570 // splitting.
571 if (CC == CallingConv::AMDGPU_KERNEL)
572 return lowerFormalArgumentsKernel(B, F, VRegs);
573
574 const bool IsGraphics = AMDGPU::isGraphics(CC);
575 const bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC);
576
577 MachineFunction &MF = B.getMF();
578 MachineBasicBlock &MBB = B.getMBB();
579 MachineRegisterInfo &MRI = MF.getRegInfo();
580 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
581 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
582 const SIRegisterInfo *TRI = Subtarget.getRegisterInfo();
583 const DataLayout &DL = F.getParent()->getDataLayout();
584
585 Info->allocateKnownAddressLDSGlobal(F);
586
587 SmallVector<CCValAssign, 16> ArgLocs;
588 CCState CCInfo(CC, F.isVarArg(), MF, ArgLocs, F.getContext());
589
590 if (Info->hasImplicitBufferPtr()) {
591 Register ImplicitBufferPtrReg = Info->addImplicitBufferPtr(*TRI);
592 MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
593 CCInfo.AllocateReg(ImplicitBufferPtrReg);
594 }
595
596 // FIXME: This probably isn't defined for mesa
597 if (Info->hasFlatScratchInit() && !Subtarget.isAmdPalOS()) {
598 Register FlatScratchInitReg = Info->addFlatScratchInit(*TRI);
599 MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
600 CCInfo.AllocateReg(FlatScratchInitReg);
601 }
602
603 SmallVector<ArgInfo, 32> SplitArgs;
604 unsigned Idx = 0;
605 unsigned PSInputNum = 0;
606
607 // Insert the hidden sret parameter if the return value won't fit in the
608 // return registers.
609 if (!FLI.CanLowerReturn)
610 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL);
611
612 for (auto &Arg : F.args()) {
613 if (DL.getTypeStoreSize(Arg.getType()) == 0)
614 continue;
615
616 const bool InReg = Arg.hasAttribute(Attribute::InReg);
617
618 // SGPR arguments to functions not implemented.
619 if (!IsGraphics && InReg)
620 return false;
621
622 if (Arg.hasAttribute(Attribute::SwiftSelf) ||
623 Arg.hasAttribute(Attribute::SwiftError) ||
624 Arg.hasAttribute(Attribute::Nest))
625 return false;
626
627 if (CC == CallingConv::AMDGPU_PS && !InReg && PSInputNum <= 15) {
628 const bool ArgUsed = !Arg.use_empty();
629 bool SkipArg = !ArgUsed && !Info->isPSInputAllocated(PSInputNum);
630
631 if (!SkipArg) {
632 Info->markPSInputAllocated(PSInputNum);
633 if (ArgUsed)
634 Info->markPSInputEnabled(PSInputNum);
635 }
636
637 ++PSInputNum;
638
639 if (SkipArg) {
640 for (Register R : VRegs[Idx])
641 B.buildUndef(R);
642
643 ++Idx;
644 continue;
645 }
646 }
647
648 ArgInfo OrigArg(VRegs[Idx], Arg, Idx);
649 const unsigned OrigArgIdx = Idx + AttributeList::FirstArgIndex;
650 setArgFlags(OrigArg, OrigArgIdx, DL, F);
651
652 splitToValueTypes(OrigArg, SplitArgs, DL, CC);
653 ++Idx;
654 }
655
656 // At least one interpolation mode must be enabled or else the GPU will
657 // hang.
658 //
659 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
660 // set PSInputAddr, the user wants to enable some bits after the compilation
661 // based on run-time states. Since we can't know what the final PSInputEna
662 // will look like, so we shouldn't do anything here and the user should take
663 // responsibility for the correct programming.
664 //
665 // Otherwise, the following restrictions apply:
666 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
667 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
668 // enabled too.
669 if (CC == CallingConv::AMDGPU_PS) {
670 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
671 ((Info->getPSInputAddr() & 0xF) == 0 &&
672 Info->isPSInputAllocated(11))) {
673 CCInfo.AllocateReg(AMDGPU::VGPR0);
674 CCInfo.AllocateReg(AMDGPU::VGPR1);
675 Info->markPSInputAllocated(0);
676 Info->markPSInputEnabled(0);
677 }
678
679 if (Subtarget.isAmdPalOS()) {
680 // For isAmdPalOS, the user does not enable some bits after compilation
681 // based on run-time states; the register values being generated here are
682 // the final ones set in hardware. Therefore we need to apply the
683 // workaround to PSInputAddr and PSInputEnable together. (The case where
684 // a bit is set in PSInputAddr but not PSInputEnable is where the frontend
685 // set up an input arg for a particular interpolation mode, but nothing
686 // uses that input arg. Really we should have an earlier pass that removes
687 // such an arg.)
688 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
689 if ((PsInputBits & 0x7F) == 0 ||
690 ((PsInputBits & 0xF) == 0 &&
691 (PsInputBits >> 11 & 1)))
692 Info->markPSInputEnabled(
693 countTrailingZeros(Info->getPSInputAddr(), ZB_Undefined));
694 }
695 }
696
697 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
698 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CC, F.isVarArg());
699
700 if (!MBB.empty())
701 B.setInstr(*MBB.begin());
702
703 if (!IsEntryFunc && !IsGraphics) {
704 // For the fixed ABI, pass workitem IDs in the last argument register.
705 TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
706 }
707
708 IncomingValueAssigner Assigner(AssignFn);
709 if (!determineAssignments(Assigner, SplitArgs, CCInfo))
710 return false;
711
712 FormalArgHandler Handler(B, MRI);
713 if (!handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, B))
714 return false;
715
716 uint64_t StackOffset = Assigner.StackOffset;
717
718 // Start adding system SGPRs.
719 if (IsEntryFunc) {
720 TLI.allocateSystemSGPRs(CCInfo, MF, *Info, CC, IsGraphics);
721 } else {
722 if (!Subtarget.enableFlatScratch())
723 CCInfo.AllocateReg(Info->getScratchRSrcReg());
724 TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
725 }
726
727 // When we tail call, we need to check if the callee's arguments will fit on
728 // the caller's stack. So, whenever we lower formal arguments, we should keep
729 // track of this information, since we might lower a tail call in this
730 // function later.
731 Info->setBytesInStackArgArea(StackOffset);
732
733 // Move back to the end of the basic block.
734 B.setMBB(MBB);
735
736 return true;
737}
738
739bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
740 CCState &CCInfo,
741 SmallVectorImpl<std::pair<MCRegister, Register>> &ArgRegs,
742 CallLoweringInfo &Info) const {
743 MachineFunction &MF = MIRBuilder.getMF();
744
745 // If there's no call site, this doesn't correspond to a call from the IR and
746 // doesn't need implicit inputs.
747 if (!Info.CB)
748 return true;
749
750 const AMDGPUFunctionArgInfo *CalleeArgInfo
751 = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
752
753 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
754 const AMDGPUFunctionArgInfo &CallerArgInfo = MFI->getArgInfo();
755
756
757 // TODO: Unify with private memory register handling. This is complicated by
758 // the fact that at least in kernels, the input argument is not necessarily
759 // in the same location as the input.
760 AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = {
761 AMDGPUFunctionArgInfo::DISPATCH_PTR,
762 AMDGPUFunctionArgInfo::QUEUE_PTR,
763 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR,
764 AMDGPUFunctionArgInfo::DISPATCH_ID,
765 AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
766 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
767 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
768 AMDGPUFunctionArgInfo::LDS_KERNEL_ID,
769 };
770
771 static constexpr StringLiteral ImplicitAttrNames[] = {
772 "amdgpu-no-dispatch-ptr",
773 "amdgpu-no-queue-ptr",
774 "amdgpu-no-implicitarg-ptr",
775 "amdgpu-no-dispatch-id",
776 "amdgpu-no-workgroup-id-x",
777 "amdgpu-no-workgroup-id-y",
778 "amdgpu-no-workgroup-id-z",
779 "amdgpu-no-lds-kernel-id",
780 };
781
782 MachineRegisterInfo &MRI = MF.getRegInfo();
783
784 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
785 const AMDGPULegalizerInfo *LI
786 = static_cast<const AMDGPULegalizerInfo*>(ST.getLegalizerInfo());
787
788 unsigned I = 0;
789 for (auto InputID : InputRegs) {
790 const ArgDescriptor *OutgoingArg;
791 const TargetRegisterClass *ArgRC;
792 LLT ArgTy;
793
794 // If the callee does not use the attribute value, skip copying the value.
795 if (Info.CB->hasFnAttr(ImplicitAttrNames[I++]))
796 continue;
797
798 std::tie(OutgoingArg, ArgRC, ArgTy) =
799 CalleeArgInfo->getPreloadedValue(InputID);
800 if (!OutgoingArg)
801 continue;
802
803 const ArgDescriptor *IncomingArg;
804 const TargetRegisterClass *IncomingArgRC;
805 std::tie(IncomingArg, IncomingArgRC, ArgTy) =
806 CallerArgInfo.getPreloadedValue(InputID);
807 assert(IncomingArgRC == ArgRC)(static_cast <bool> (IncomingArgRC == ArgRC) ? void (0)
: __assert_fail ("IncomingArgRC == ArgRC", "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp"
, 807, __extension__ __PRETTY_FUNCTION__))
;
808
809 Register InputReg = MRI.createGenericVirtualRegister(ArgTy);
810
811 if (IncomingArg) {
812 LI->loadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);
813 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
814 LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder);
815 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
816 Optional<uint32_t> Id =
817 AMDGPUMachineFunction::getLDSKernelIdMetadata(MF.getFunction());
818 if (Id.has_value()) {
819 MIRBuilder.buildConstant(InputReg, Id.value());
820 } else {
821 MIRBuilder.buildUndef(InputReg);
822 }
823 } else {
824 // We may have proven the input wasn't needed, although the ABI is
825 // requiring it. We just need to allocate the register appropriately.
826 MIRBuilder.buildUndef(InputReg);
827 }
828
829 if (OutgoingArg->isRegister()) {
830 ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);
831 if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
832 report_fatal_error("failed to allocate implicit input argument");
833 } else {
834 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Unhandled stack passed implicit input argument\n"
; } } while (false)
;
835 return false;
836 }
837 }
838
839 // Pack workitem IDs into a single register or pass it as is if already
840 // packed.
841 const ArgDescriptor *OutgoingArg;
842 const TargetRegisterClass *ArgRC;
843 LLT ArgTy;
844
845 std::tie(OutgoingArg, ArgRC, ArgTy) =
846 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
847 if (!OutgoingArg)
848 std::tie(OutgoingArg, ArgRC, ArgTy) =
849 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
850 if (!OutgoingArg)
851 std::tie(OutgoingArg, ArgRC, ArgTy) =
852 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
853 if (!OutgoingArg)
854 return false;
855
856 auto WorkitemIDX =
857 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
858 auto WorkitemIDY =
859 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
860 auto WorkitemIDZ =
861 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
862
863 const ArgDescriptor *IncomingArgX = std::get<0>(WorkitemIDX);
864 const ArgDescriptor *IncomingArgY = std::get<0>(WorkitemIDY);
865 const ArgDescriptor *IncomingArgZ = std::get<0>(WorkitemIDZ);
866 const LLT S32 = LLT::scalar(32);
867
868 const bool NeedWorkItemIDX = !Info.CB->hasFnAttr("amdgpu-no-workitem-id-x");
869 const bool NeedWorkItemIDY = !Info.CB->hasFnAttr("amdgpu-no-workitem-id-y");
870 const bool NeedWorkItemIDZ = !Info.CB->hasFnAttr("amdgpu-no-workitem-id-z");
871
872 // If incoming ids are not packed we need to pack them.
873 // FIXME: Should consider known workgroup size to eliminate known 0 cases.
874 Register InputReg;
875 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX &&
876 NeedWorkItemIDX) {
877 if (ST.getMaxWorkitemID(MF.getFunction(), 0) != 0) {
878 InputReg = MRI.createGenericVirtualRegister(S32);
879 LI->loadInputValue(InputReg, MIRBuilder, IncomingArgX,
880 std::get<1>(WorkitemIDX), std::get<2>(WorkitemIDX));
881 } else {
882 InputReg = MIRBuilder.buildConstant(S32, 0).getReg(0);
883 }
884 }
885
886 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY &&
887 NeedWorkItemIDY && ST.getMaxWorkitemID(MF.getFunction(), 1) != 0) {
888 Register Y = MRI.createGenericVirtualRegister(S32);
889 LI->loadInputValue(Y, MIRBuilder, IncomingArgY, std::get<1>(WorkitemIDY),
890 std::get<2>(WorkitemIDY));
891
892 Y = MIRBuilder.buildShl(S32, Y, MIRBuilder.buildConstant(S32, 10)).getReg(0);
893 InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Y).getReg(0) : Y;
894 }
895
896 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ &&
897 NeedWorkItemIDZ && ST.getMaxWorkitemID(MF.getFunction(), 2) != 0) {
898 Register Z = MRI.createGenericVirtualRegister(S32);
899 LI->loadInputValue(Z, MIRBuilder, IncomingArgZ, std::get<1>(WorkitemIDZ),
900 std::get<2>(WorkitemIDZ));
901
902 Z = MIRBuilder.buildShl(S32, Z, MIRBuilder.buildConstant(S32, 20)).getReg(0);
903 InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Z).getReg(0) : Z;
904 }
905
906 if (!InputReg &&
907 (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
908 InputReg = MRI.createGenericVirtualRegister(S32);
909 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
910 // We're in a situation where the outgoing function requires the workitem
911 // ID, but the calling function does not have it (e.g a graphics function
912 // calling a C calling convention function). This is illegal, but we need
913 // to produce something.
914 MIRBuilder.buildUndef(InputReg);
915 } else {
916 // Workitem ids are already packed, any of present incoming arguments will
917 // carry all required fields.
918 ArgDescriptor IncomingArg = ArgDescriptor::createArg(
919 IncomingArgX ? *IncomingArgX :
920 IncomingArgY ? *IncomingArgY : *IncomingArgZ, ~0u);
921 LI->loadInputValue(InputReg, MIRBuilder, &IncomingArg,
922 &AMDGPU::VGPR_32RegClass, S32);
923 }
924 }
925
926 if (OutgoingArg->isRegister()) {
927 if (InputReg)
928 ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);
929
930 if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
931 report_fatal_error("failed to allocate implicit input argument");
932 } else {
933 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Unhandled stack passed implicit input argument\n"
; } } while (false)
;
934 return false;
935 }
936
937 return true;
938}
939
940/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
941/// CC.
942static std::pair<CCAssignFn *, CCAssignFn *>
943getAssignFnsForCC(CallingConv::ID CC, const SITargetLowering &TLI) {
944 return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
945}
946
947static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
948 bool IsTailCall) {
949 assert(!(IsIndirect && IsTailCall) && "Indirect calls can't be tail calls, "(static_cast <bool> (!(IsIndirect && IsTailCall
) && "Indirect calls can't be tail calls, " "because the address can be divergent"
) ? void (0) : __assert_fail ("!(IsIndirect && IsTailCall) && \"Indirect calls can't be tail calls, \" \"because the address can be divergent\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 950, __extension__
__PRETTY_FUNCTION__))
950 "because the address can be divergent")(static_cast <bool> (!(IsIndirect && IsTailCall
) && "Indirect calls can't be tail calls, " "because the address can be divergent"
) ? void (0) : __assert_fail ("!(IsIndirect && IsTailCall) && \"Indirect calls can't be tail calls, \" \"because the address can be divergent\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 950, __extension__
__PRETTY_FUNCTION__))
;
951 return IsTailCall ? AMDGPU::SI_TCRETURN : AMDGPU::G_SI_CALL;
952}
953
954// Add operands to call instruction to track the callee.
955static bool addCallTargetOperands(MachineInstrBuilder &CallInst,
956 MachineIRBuilder &MIRBuilder,
957 AMDGPUCallLowering::CallLoweringInfo &Info) {
958 if (Info.Callee.isReg()) {
959 CallInst.addReg(Info.Callee.getReg());
960 CallInst.addImm(0);
961 } else if (Info.Callee.isGlobal() && Info.Callee.getOffset() == 0) {
962 // The call lowering lightly assumed we can directly encode a call target in
963 // the instruction, which is not the case. Materialize the address here.
964 const GlobalValue *GV = Info.Callee.getGlobal();
965 auto Ptr = MIRBuilder.buildGlobalValue(
966 LLT::pointer(GV->getAddressSpace(), 64), GV);
967 CallInst.addReg(Ptr.getReg(0));
968 CallInst.add(Info.Callee);
969 } else
970 return false;
971
972 return true;
973}
974
975bool AMDGPUCallLowering::doCallerAndCalleePassArgsTheSameWay(
976 CallLoweringInfo &Info, MachineFunction &MF,
977 SmallVectorImpl<ArgInfo> &InArgs) const {
978 const Function &CallerF = MF.getFunction();
979 CallingConv::ID CalleeCC = Info.CallConv;
980 CallingConv::ID CallerCC = CallerF.getCallingConv();
981
982 // If the calling conventions match, then everything must be the same.
983 if (CalleeCC == CallerCC)
984 return true;
985
986 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
987
988 // Make sure that the caller and callee preserve all of the same registers.
989 auto TRI = ST.getRegisterInfo();
990
991 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
992 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
993 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
994 return false;
995
996 // Check if the caller and callee will handle arguments in the same way.
997 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
998 CCAssignFn *CalleeAssignFnFixed;
999 CCAssignFn *CalleeAssignFnVarArg;
1000 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
1001 getAssignFnsForCC(CalleeCC, TLI);
1002
1003 CCAssignFn *CallerAssignFnFixed;
1004 CCAssignFn *CallerAssignFnVarArg;
1005 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
1006 getAssignFnsForCC(CallerCC, TLI);
1007
1008 // FIXME: We are not accounting for potential differences in implicitly passed
1009 // inputs, but only the fixed ABI is supported now anyway.
1010 IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
1011 CalleeAssignFnVarArg);
1012 IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
1013 CallerAssignFnVarArg);
1014 return resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner);
1015}
1016
1017bool AMDGPUCallLowering::areCalleeOutgoingArgsTailCallable(
1018 CallLoweringInfo &Info, MachineFunction &MF,
1019 SmallVectorImpl<ArgInfo> &OutArgs) const {
1020 // If there are no outgoing arguments, then we are done.
1021 if (OutArgs.empty())
1022 return true;
1023
1024 const Function &CallerF = MF.getFunction();
1025 CallingConv::ID CalleeCC = Info.CallConv;
1026 CallingConv::ID CallerCC = CallerF.getCallingConv();
1027 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1028
1029 CCAssignFn *AssignFnFixed;
1030 CCAssignFn *AssignFnVarArg;
1031 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
1032
1033 // We have outgoing arguments. Make sure that we can tail call with them.
1034 SmallVector<CCValAssign, 16> OutLocs;
1035 CCState OutInfo(CalleeCC, false, MF, OutLocs, CallerF.getContext());
1036 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1037
1038 if (!determineAssignments(Assigner, OutArgs, OutInfo)) {
1039 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Could not analyze call operands.\n"
; } } while (false)
;
1040 return false;
1041 }
1042
1043 // Make sure that they can fit on the caller's stack.
1044 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1045 if (OutInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) {
1046 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Cannot fit call operands on caller's stack.\n"
; } } while (false)
;
1047 return false;
1048 }
1049
1050 // Verify that the parameters in callee-saved registers match.
1051 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1052 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1053 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
1054 MachineRegisterInfo &MRI = MF.getRegInfo();
1055 return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);
1056}
1057
1058/// Return true if the calling convention is one that we can guarantee TCO for.
1059static bool canGuaranteeTCO(CallingConv::ID CC) {
1060 return CC == CallingConv::Fast;
1061}
1062
1063/// Return true if we might ever do TCO for calls with this calling convention.
1064static bool mayTailCallThisCC(CallingConv::ID CC) {
1065 switch (CC) {
1066 case CallingConv::C:
1067 case CallingConv::AMDGPU_Gfx:
1068 return true;
1069 default:
1070 return canGuaranteeTCO(CC);
1071 }
1072}
1073
1074bool AMDGPUCallLowering::isEligibleForTailCallOptimization(
1075 MachineIRBuilder &B, CallLoweringInfo &Info,
1076 SmallVectorImpl<ArgInfo> &InArgs, SmallVectorImpl<ArgInfo> &OutArgs) const {
1077 // Must pass all target-independent checks in order to tail call optimize.
1078 if (!Info.IsTailCall)
1079 return false;
1080
1081 // Indirect calls can't be tail calls, because the address can be divergent.
1082 // TODO Check divergence info if the call really is divergent.
1083 if (Info.Callee.isReg())
1084 return false;
1085
1086 MachineFunction &MF = B.getMF();
1087 const Function &CallerF = MF.getFunction();
1088 CallingConv::ID CalleeCC = Info.CallConv;
1089 CallingConv::ID CallerCC = CallerF.getCallingConv();
1090
1091 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1092 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
1093 // Kernels aren't callable, and don't have a live in return address so it
1094 // doesn't make sense to do a tail call with entry functions.
1095 if (!CallerPreserved)
1096 return false;
1097
1098 if (!mayTailCallThisCC(CalleeCC)) {
1099 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Calling convention cannot be tail called.\n"
; } } while (false)
;
1100 return false;
1101 }
1102
1103 if (any_of(CallerF.args(), [](const Argument &A) {
1104 return A.hasByValAttr() || A.hasSwiftErrorAttr();
1105 })) {
1106 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Cannot tail call from callers with byval "
"or swifterror arguments\n"; } } while (false)
1107 "or swifterror arguments\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Cannot tail call from callers with byval "
"or swifterror arguments\n"; } } while (false)
;
1108 return false;
1109 }
1110
1111 // If we have -tailcallopt, then we're done.
1112 if (MF.getTarget().Options.GuaranteedTailCallOpt)
1113 return canGuaranteeTCO(CalleeCC) && CalleeCC == CallerF.getCallingConv();
1114
1115 // Verify that the incoming and outgoing arguments from the callee are
1116 // safe to tail call.
1117 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
1118 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
1119 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
1120 << "... Caller and callee have incompatible calling conventions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
;
1121 return false;
1122 }
1123
1124 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
1125 return false;
1126
1127 LLVM_DEBUG(dbgs() << "... Call is eligible for tail call optimization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "... Call is eligible for tail call optimization.\n"
; } } while (false)
;
1128 return true;
1129}
1130
1131// Insert outgoing implicit arguments for a call, by inserting copies to the
1132// implicit argument registers and adding the necessary implicit uses to the
1133// call instruction.
1134void AMDGPUCallLowering::handleImplicitCallArguments(
1135 MachineIRBuilder &MIRBuilder, MachineInstrBuilder &CallInst,
1136 const GCNSubtarget &ST, const SIMachineFunctionInfo &FuncInfo,
1137 ArrayRef<std::pair<MCRegister, Register>> ImplicitArgRegs) const {
1138 if (!ST.enableFlatScratch()) {
1139 // Insert copies for the SRD. In the HSA case, this should be an identity
1140 // copy.
1141 auto ScratchRSrcReg = MIRBuilder.buildCopy(LLT::fixed_vector(4, 32),
1142 FuncInfo.getScratchRSrcReg());
1143 MIRBuilder.buildCopy(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
1144 CallInst.addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Implicit);
1145 }
1146
1147 for (std::pair<MCRegister, Register> ArgReg : ImplicitArgRegs) {
1148 MIRBuilder.buildCopy((Register)ArgReg.first, ArgReg.second);
1149 CallInst.addReg(ArgReg.first, RegState::Implicit);
1150 }
1151}
1152
1153bool AMDGPUCallLowering::lowerTailCall(
1154 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1155 SmallVectorImpl<ArgInfo> &OutArgs) const {
1156 MachineFunction &MF = MIRBuilder.getMF();
1157 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1158 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1159 const Function &F = MF.getFunction();
1160 MachineRegisterInfo &MRI = MF.getRegInfo();
1161 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1162
1163 // True when we're tail calling, but without -tailcallopt.
1164 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt;
1165
1166 // Find out which ABI gets to decide where things go.
1167 CallingConv::ID CalleeCC = Info.CallConv;
1168 CCAssignFn *AssignFnFixed;
1169 CCAssignFn *AssignFnVarArg;
1170 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
1171
1172 MachineInstrBuilder CallSeqStart;
1173 if (!IsSibCall)
1174 CallSeqStart = MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP);
1175
1176 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true);
1177 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1178 if (!addCallTargetOperands(MIB, MIRBuilder, Info))
1179 return false;
1180
1181 // Byte offset for the tail call. When we are sibcalling, this will always
1182 // be 0.
1183 MIB.addImm(0);
1184
1185 // Tell the call which registers are clobbered.
1186 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1187 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1188 MIB.addRegMask(Mask);
1189
1190 // FPDiff is the byte offset of the call's argument area from the callee's.
1191 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1192 // by this amount for a tail call. In a sibling call it must be 0 because the
1193 // caller will deallocate the entire stack and the callee still expects its
1194 // arguments to begin at SP+0.
1195 int FPDiff = 0;
1196
1197 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1198 // by -tailcallopt. For sibcalls, the memory operands for the call are
1199 // already available in the caller's incoming argument space.
1200 unsigned NumBytes = 0;
1201 if (!IsSibCall) {
1202 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1203 // before handling assignments, because FPDiff must be known for memory
1204 // arguments.
1205 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1206 SmallVector<CCValAssign, 16> OutLocs;
1207 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1208
1209 // FIXME: Not accounting for callee implicit inputs
1210 OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg);
1211 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo))
1212 return false;
1213
1214 // The callee will pop the argument stack as a tail call. Thus, we must
1215 // keep it 16-byte aligned.
1216 NumBytes = alignTo(OutInfo.getNextStackOffset(), ST.getStackAlignment());
1217
1218 // FPDiff will be negative if this tail call requires more space than we
1219 // would automatically have in our incoming argument space. Positive if we
1220 // actually shrink the stack.
1221 FPDiff = NumReusableBytes - NumBytes;
1222
1223 // The stack pointer must be 16-byte aligned at all times it's used for a
1224 // memory operation, which in practice means at *all* times and in
1225 // particular across call boundaries. Therefore our own arguments started at
1226 // a 16-byte aligned SP and the delta applied for the tail call should
1227 // satisfy the same constraint.
1228 assert(isAligned(ST.getStackAlignment(), FPDiff) &&(static_cast <bool> (isAligned(ST.getStackAlignment(), FPDiff
) && "unaligned stack on tail call") ? void (0) : __assert_fail
("isAligned(ST.getStackAlignment(), FPDiff) && \"unaligned stack on tail call\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 1229, __extension__
__PRETTY_FUNCTION__))
1229 "unaligned stack on tail call")(static_cast <bool> (isAligned(ST.getStackAlignment(), FPDiff
) && "unaligned stack on tail call") ? void (0) : __assert_fail
("isAligned(ST.getStackAlignment(), FPDiff) && \"unaligned stack on tail call\""
, "llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp", 1229, __extension__
__PRETTY_FUNCTION__))
;
1230 }
1231
1232 SmallVector<CCValAssign, 16> ArgLocs;
1233 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());
1234
1235 // We could pass MIB and directly add the implicit uses to the call
1236 // now. However, as an aesthetic choice, place implicit argument operands
1237 // after the ordinary user argument registers.
1238 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
1239
1240 if (Info.CallConv != CallingConv::AMDGPU_Gfx) {
1241 // With a fixed ABI, allocate fixed registers before user arguments.
1242 if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))
1243 return false;
1244 }
1245
1246 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1247
1248 if (!determineAssignments(Assigner, OutArgs, CCInfo))
1249 return false;
1250
1251 // Do the actual argument marshalling.
1252 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, true, FPDiff);
1253 if (!handleAssignments(Handler, OutArgs, CCInfo, ArgLocs, MIRBuilder))
1254 return false;
1255
1256 handleImplicitCallArguments(MIRBuilder, MIB, ST, *FuncInfo, ImplicitArgRegs);
1257
1258 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1259 // sequence start and end here.
1260 if (!IsSibCall) {
1261 MIB->getOperand(1).setImm(FPDiff);
1262 CallSeqStart.addImm(NumBytes).addImm(0);
1263 // End the call sequence *before* emitting the call. Normally, we would
1264 // tidy the frame up after the call. However, here, we've laid out the
1265 // parameters so that when SP is reset, they will be in the correct
1266 // location.
1267 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKDOWN).addImm(NumBytes).addImm(0);
1268 }
1269
1270 // Now we can add the actual call instruction to the correct basic block.
1271 MIRBuilder.insertInstr(MIB);
1272
1273 // If Callee is a reg, since it is used by a target specific
1274 // instruction, it must have a register class matching the
1275 // constraint of that instruction.
1276
1277 // FIXME: We should define regbankselectable call instructions to handle
1278 // divergent call targets.
1279 if (MIB->getOperand(0).isReg()) {
1280 MIB->getOperand(0).setReg(constrainOperandRegClass(
1281 MF, *TRI, MRI, *ST.getInstrInfo(), *ST.getRegBankInfo(), *MIB,
1282 MIB->getDesc(), MIB->getOperand(0), 0));
1283 }
1284
1285 MF.getFrameInfo().setHasTailCall();
1286 Info.LoweredTailCall = true;
1287 return true;
1288}
1289
1290bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
1291 CallLoweringInfo &Info) const {
1292 if (Info.IsVarArg) {
1293 LLVM_DEBUG(dbgs() << "Variadic functions not implemented\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Variadic functions not implemented\n"
; } } while (false)
;
1294 return false;
1295 }
1296
1297 MachineFunction &MF = MIRBuilder.getMF();
1298 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1299 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1300
1301 const Function &F = MF.getFunction();
1302 MachineRegisterInfo &MRI = MF.getRegInfo();
1303 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1304 const DataLayout &DL = F.getParent()->getDataLayout();
1305
1306 SmallVector<ArgInfo, 8> OutArgs;
1307 for (auto &OrigArg : Info.OrigArgs)
1308 splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);
1309
1310 SmallVector<ArgInfo, 8> InArgs;
1311 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy())
1312 splitToValueTypes(Info.OrigRet, InArgs, DL, Info.CallConv);
1313
1314 // If we can lower as a tail call, do that instead.
1315 bool CanTailCallOpt =
1316 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
1317
1318 // We must emit a tail call if we have musttail.
1319 if (Info.IsMustTailCall && !CanTailCallOpt) {
1320 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-call-lowering")) { dbgs() << "Failed to lower musttail call as tail call\n"
; } } while (false)
;
1321 return false;
1322 }
1323
1324 Info.IsTailCall = CanTailCallOpt;
1325 if (CanTailCallOpt)
1326 return lowerTailCall(MIRBuilder, Info, OutArgs);
1327
1328 // Find out which ABI gets to decide where things go.
1329 CCAssignFn *AssignFnFixed;
1330 CCAssignFn *AssignFnVarArg;
1331 std::tie(AssignFnFixed, AssignFnVarArg) =
1332 getAssignFnsForCC(Info.CallConv, TLI);
1333
1334 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP)
1335 .addImm(0)
1336 .addImm(0);
1337
1338 // Create a temporarily-floating call instruction so we can add the implicit
1339 // uses of arg registers.
1340 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
1341
1342 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1343 MIB.addDef(TRI->getReturnAddressReg(MF));
1344
1345 if (!addCallTargetOperands(MIB, MIRBuilder, Info))
1346 return false;
1347
1348 // Tell the call which registers are clobbered.
1349 const uint32_t *Mask = TRI->getCallPreservedMask(MF, Info.CallConv);
1350 MIB.addRegMask(Mask);
1351
1352 SmallVector<CCValAssign, 16> ArgLocs;
1353 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());
1354
1355 // We could pass MIB and directly add the implicit uses to the call
1356 // now. However, as an aesthetic choice, place implicit argument operands
1357 // after the ordinary user argument registers.
1358 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
1359
1360 if (Info.CallConv != CallingConv::AMDGPU_Gfx) {
1361 // With a fixed ABI, allocate fixed registers before user arguments.
1362 if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))
1363 return false;
1364 }
1365
1366 // Do the actual argument marshalling.
1367 SmallVector<Register, 8> PhysRegs;
1368
1369 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1370 if (!determineAssignments(Assigner, OutArgs, CCInfo))
1371 return false;
1372
1373 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, false);
1374 if (!handleAssignments(Handler, OutArgs, CCInfo, ArgLocs, MIRBuilder))
1375 return false;
1376
1377 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1378
1379 handleImplicitCallArguments(MIRBuilder, MIB, ST, *MFI, ImplicitArgRegs);
1380
1381 // Get a count of how many bytes are to be pushed on the stack.
1382 unsigned NumBytes = CCInfo.getNextStackOffset();
1383
1384 // If Callee is a reg, since it is used by a target specific
1385 // instruction, it must have a register class matching the
1386 // constraint of that instruction.
1387
1388 // FIXME: We should define regbankselectable call instructions to handle
1389 // divergent call targets.
1390 if (MIB->getOperand(1).isReg()) {
1391 MIB->getOperand(1).setReg(constrainOperandRegClass(
1392 MF, *TRI, MRI, *ST.getInstrInfo(),
1393 *ST.getRegBankInfo(), *MIB, MIB->getDesc(), MIB->getOperand(1),
1394 1));
1395 }
1396
1397 // Now we can add the actual call instruction to the correct position.
1398 MIRBuilder.insertInstr(MIB);
1399
1400 // Finally we can copy the returned value back into its virtual-register. In
1401 // symmetry with the arguments, the physical register must be an
1402 // implicit-define of the call instruction.
1403 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1404 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv,
1405 Info.IsVarArg);
1406 IncomingValueAssigner Assigner(RetAssignFn);
1407 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1408 if (!determineAndHandleAssignments(Handler, Assigner, InArgs, MIRBuilder,
1409 Info.CallConv, Info.IsVarArg))
1410 return false;
1411 }
1412
1413 uint64_t CalleePopBytes = NumBytes;
1414
1415 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKDOWN)
1416 .addImm(0)
1417 .addImm(CalleePopBytes);
1418
1419 if (!Info.CanLowerReturn) {
1420 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,
1421 Info.DemoteRegister, Info.DemoteStackIndex);
1422 }
1423
1424 return true;
1425}