Bug Summary

File:build/source/llvm/include/llvm/Analysis/ObjCARCUtil.h
Warning:line 44, column 12
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name AArch64CallLowering.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/AArch64 -I /build/source/llvm/lib/Target/AArch64 -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1679443490 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-03-22-005342-16304-1 -x c++ /build/source/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp

/build/source/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp

1//===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AArch64CallLowering.h"
16#include "AArch64ISelLowering.h"
17#include "AArch64MachineFunctionInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/Analysis/ObjCARCUtil.h"
23#include "llvm/CodeGen/Analysis.h"
24#include "llvm/CodeGen/CallingConvLower.h"
25#include "llvm/CodeGen/FunctionLoweringInfo.h"
26#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
27#include "llvm/CodeGen/GlobalISel/Utils.h"
28#include "llvm/CodeGen/LowLevelType.h"
29#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineFrameInfo.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
33#include "llvm/CodeGen/MachineMemOperand.h"
34#include "llvm/CodeGen/MachineOperand.h"
35#include "llvm/CodeGen/MachineRegisterInfo.h"
36#include "llvm/CodeGen/TargetRegisterInfo.h"
37#include "llvm/CodeGen/TargetSubtargetInfo.h"
38#include "llvm/CodeGen/ValueTypes.h"
39#include "llvm/IR/Argument.h"
40#include "llvm/IR/Attributes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Type.h"
43#include "llvm/IR/Value.h"
44#include "llvm/Support/MachineValueType.h"
45#include <algorithm>
46#include <cassert>
47#include <cstdint>
48#include <iterator>
49
50#define DEBUG_TYPE"aarch64-call-lowering" "aarch64-call-lowering"
51
52using namespace llvm;
53
54AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
55 : CallLowering(&TLI) {}
56
57static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT,
58 MVT &LocVT) {
59 // If ValVT is i1/i8/i16, we should set LocVT to i8/i8/i16. This is a legacy
60 // hack because the DAG calls the assignment function with pre-legalized
61 // register typed values, not the raw type.
62 //
63 // This hack is not applied to return values which are not passed on the
64 // stack.
65 if (OrigVT == MVT::i1 || OrigVT == MVT::i8)
66 ValVT = LocVT = MVT::i8;
67 else if (OrigVT == MVT::i16)
68 ValVT = LocVT = MVT::i16;
69}
70
71// Account for i1/i8/i16 stack passed value hack
72static LLT getStackValueStoreTypeHack(const CCValAssign &VA) {
73 const MVT ValVT = VA.getValVT();
74 return (ValVT == MVT::i8 || ValVT == MVT::i16) ? LLT(ValVT)
75 : LLT(VA.getLocVT());
76}
77
78namespace {
79
80struct AArch64IncomingValueAssigner
81 : public CallLowering::IncomingValueAssigner {
82 AArch64IncomingValueAssigner(CCAssignFn *AssignFn_,
83 CCAssignFn *AssignFnVarArg_)
84 : IncomingValueAssigner(AssignFn_, AssignFnVarArg_) {}
85
86 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
87 CCValAssign::LocInfo LocInfo,
88 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
89 CCState &State) override {
90 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
91 return IncomingValueAssigner::assignArg(ValNo, OrigVT, ValVT, LocVT,
92 LocInfo, Info, Flags, State);
93 }
94};
95
96struct AArch64OutgoingValueAssigner
97 : public CallLowering::OutgoingValueAssigner {
98 const AArch64Subtarget &Subtarget;
99
100 /// Track if this is used for a return instead of function argument
101 /// passing. We apply a hack to i1/i8/i16 stack passed values, but do not use
102 /// stack passed returns for them and cannot apply the type adjustment.
103 bool IsReturn;
104
105 AArch64OutgoingValueAssigner(CCAssignFn *AssignFn_,
106 CCAssignFn *AssignFnVarArg_,
107 const AArch64Subtarget &Subtarget_,
108 bool IsReturn)
109 : OutgoingValueAssigner(AssignFn_, AssignFnVarArg_),
110 Subtarget(Subtarget_), IsReturn(IsReturn) {}
111
112 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
113 CCValAssign::LocInfo LocInfo,
114 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
115 CCState &State) override {
116 bool IsCalleeWin = Subtarget.isCallingConvWin64(State.getCallingConv());
117 bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
118
119 bool Res;
120 if (Info.IsFixed && !UseVarArgsCCForFixed) {
121 if (!IsReturn)
122 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
123 Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
124 } else
125 Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, State);
126
127 StackOffset = State.getNextStackOffset();
128 return Res;
129 }
130};
131
132struct IncomingArgHandler : public CallLowering::IncomingValueHandler {
133 IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
134 : IncomingValueHandler(MIRBuilder, MRI) {}
135
136 Register getStackAddress(uint64_t Size, int64_t Offset,
137 MachinePointerInfo &MPO,
138 ISD::ArgFlagsTy Flags) override {
139 auto &MFI = MIRBuilder.getMF().getFrameInfo();
140
141 // Byval is assumed to be writable memory, but other stack passed arguments
142 // are not.
143 const bool IsImmutable = !Flags.isByVal();
144
145 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);
146 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
147 auto AddrReg = MIRBuilder.buildFrameIndex(LLT::pointer(0, 64), FI);
148 return AddrReg.getReg(0);
149 }
150
151 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
152 ISD::ArgFlagsTy Flags) const override {
153 // For pointers, we just need to fixup the integer types reported in the
154 // CCValAssign.
155 if (Flags.isPointer())
156 return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
157 return getStackValueStoreTypeHack(VA);
158 }
159
160 void assignValueToReg(Register ValVReg, Register PhysReg,
161 CCValAssign VA) override {
162 markPhysRegUsed(PhysReg);
163 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
164 }
165
166 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
167 MachinePointerInfo &MPO, CCValAssign &VA) override {
168 MachineFunction &MF = MIRBuilder.getMF();
169
170 LLT ValTy(VA.getValVT());
171 LLT LocTy(VA.getLocVT());
172
173 // Fixup the types for the DAG compatibility hack.
174 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16)
175 std::swap(ValTy, LocTy);
176 else {
177 // The calling code knows if this is a pointer or not, we're only touching
178 // the LocTy for the i8/i16 hack.
179 assert(LocTy.getSizeInBits() == MemTy.getSizeInBits())(static_cast <bool> (LocTy.getSizeInBits() == MemTy.getSizeInBits
()) ? void (0) : __assert_fail ("LocTy.getSizeInBits() == MemTy.getSizeInBits()"
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 179
, __extension__ __PRETTY_FUNCTION__))
;
180 LocTy = MemTy;
181 }
182
183 auto MMO = MF.getMachineMemOperand(
184 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, LocTy,
185 inferAlignFromPtrInfo(MF, MPO));
186
187 switch (VA.getLocInfo()) {
188 case CCValAssign::LocInfo::ZExt:
189 MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, ValVReg, Addr, *MMO);
190 return;
191 case CCValAssign::LocInfo::SExt:
192 MIRBuilder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, ValVReg, Addr, *MMO);
193 return;
194 default:
195 MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
196 return;
197 }
198 }
199
200 /// How the physical register gets marked varies between formal
201 /// parameters (it's a basic-block live-in), and a call instruction
202 /// (it's an implicit-def of the BL).
203 virtual void markPhysRegUsed(MCRegister PhysReg) = 0;
204};
205
206struct FormalArgHandler : public IncomingArgHandler {
207 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
208 : IncomingArgHandler(MIRBuilder, MRI) {}
209
210 void markPhysRegUsed(MCRegister PhysReg) override {
211 MIRBuilder.getMRI()->addLiveIn(PhysReg);
212 MIRBuilder.getMBB().addLiveIn(PhysReg);
213 }
214};
215
216struct CallReturnHandler : public IncomingArgHandler {
217 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
218 MachineInstrBuilder MIB)
219 : IncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
220
221 void markPhysRegUsed(MCRegister PhysReg) override {
222 MIB.addDef(PhysReg, RegState::Implicit);
223 }
224
225 MachineInstrBuilder MIB;
226};
227
228/// A special return arg handler for "returned" attribute arg calls.
229struct ReturnedArgCallReturnHandler : public CallReturnHandler {
230 ReturnedArgCallReturnHandler(MachineIRBuilder &MIRBuilder,
231 MachineRegisterInfo &MRI,
232 MachineInstrBuilder MIB)
233 : CallReturnHandler(MIRBuilder, MRI, MIB) {}
234
235 void markPhysRegUsed(MCRegister PhysReg) override {}
236};
237
238struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
239 OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
240 MachineInstrBuilder MIB, bool IsTailCall = false,
241 int FPDiff = 0)
242 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), IsTailCall(IsTailCall),
243 FPDiff(FPDiff),
244 Subtarget(MIRBuilder.getMF().getSubtarget<AArch64Subtarget>()) {}
245
246 Register getStackAddress(uint64_t Size, int64_t Offset,
247 MachinePointerInfo &MPO,
248 ISD::ArgFlagsTy Flags) override {
249 MachineFunction &MF = MIRBuilder.getMF();
250 LLT p0 = LLT::pointer(0, 64);
251 LLT s64 = LLT::scalar(64);
252
253 if (IsTailCall) {
254 assert(!Flags.isByVal() && "byval unhandled with tail calls")(static_cast <bool> (!Flags.isByVal() && "byval unhandled with tail calls"
) ? void (0) : __assert_fail ("!Flags.isByVal() && \"byval unhandled with tail calls\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 254
, __extension__ __PRETTY_FUNCTION__))
;
255
256 Offset += FPDiff;
257 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
258 auto FIReg = MIRBuilder.buildFrameIndex(p0, FI);
259 MPO = MachinePointerInfo::getFixedStack(MF, FI);
260 return FIReg.getReg(0);
261 }
262
263 if (!SPReg)
264 SPReg = MIRBuilder.buildCopy(p0, Register(AArch64::SP)).getReg(0);
265
266 auto OffsetReg = MIRBuilder.buildConstant(s64, Offset);
267
268 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg);
269
270 MPO = MachinePointerInfo::getStack(MF, Offset);
271 return AddrReg.getReg(0);
272 }
273
274 /// We need to fixup the reported store size for certain value types because
275 /// we invert the interpretation of ValVT and LocVT in certain cases. This is
276 /// for compatability with the DAG call lowering implementation, which we're
277 /// currently building on top of.
278 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
279 ISD::ArgFlagsTy Flags) const override {
280 if (Flags.isPointer())
281 return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
282 return getStackValueStoreTypeHack(VA);
283 }
284
285 void assignValueToReg(Register ValVReg, Register PhysReg,
286 CCValAssign VA) override {
287 MIB.addUse(PhysReg, RegState::Implicit);
288 Register ExtReg = extendRegister(ValVReg, VA);
289 MIRBuilder.buildCopy(PhysReg, ExtReg);
290 }
291
292 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
293 MachinePointerInfo &MPO, CCValAssign &VA) override {
294 MachineFunction &MF = MIRBuilder.getMF();
295 auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,
296 inferAlignFromPtrInfo(MF, MPO));
297 MIRBuilder.buildStore(ValVReg, Addr, *MMO);
298 }
299
300 void assignValueToAddress(const CallLowering::ArgInfo &Arg, unsigned RegIndex,
301 Register Addr, LLT MemTy, MachinePointerInfo &MPO,
302 CCValAssign &VA) override {
303 unsigned MaxSize = MemTy.getSizeInBytes() * 8;
304 // For varargs, we always want to extend them to 8 bytes, in which case
305 // we disable setting a max.
306 if (!Arg.IsFixed)
307 MaxSize = 0;
308
309 Register ValVReg = Arg.Regs[RegIndex];
310 if (VA.getLocInfo() != CCValAssign::LocInfo::FPExt) {
311 MVT LocVT = VA.getLocVT();
312 MVT ValVT = VA.getValVT();
313
314 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16) {
315 std::swap(ValVT, LocVT);
316 MemTy = LLT(VA.getValVT());
317 }
318
319 ValVReg = extendRegister(ValVReg, VA, MaxSize);
320 } else {
321 // The store does not cover the full allocated stack slot.
322 MemTy = LLT(VA.getValVT());
323 }
324
325 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
326 }
327
328 MachineInstrBuilder MIB;
329
330 bool IsTailCall;
331
332 /// For tail calls, the byte offset of the call's argument area from the
333 /// callee's. Unused elsewhere.
334 int FPDiff;
335
336 // Cache the SP register vreg if we need it more than once in this call site.
337 Register SPReg;
338
339 const AArch64Subtarget &Subtarget;
340};
341} // namespace
342
343static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
344 return (CallConv == CallingConv::Fast && TailCallOpt) ||
345 CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;
346}
347
348bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
349 const Value *Val,
350 ArrayRef<Register> VRegs,
351 FunctionLoweringInfo &FLI,
352 Register SwiftErrorVReg) const {
353 auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR);
354 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&(static_cast <bool> (((Val && !VRegs.empty()) ||
(!Val && VRegs.empty())) && "Return value without a vreg"
) ? void (0) : __assert_fail ("((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && \"Return value without a vreg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 355
, __extension__ __PRETTY_FUNCTION__))
355 "Return value without a vreg")(static_cast <bool> (((Val && !VRegs.empty()) ||
(!Val && VRegs.empty())) && "Return value without a vreg"
) ? void (0) : __assert_fail ("((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && \"Return value without a vreg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 355
, __extension__ __PRETTY_FUNCTION__))
;
356
357 bool Success = true;
358 if (!FLI.CanLowerReturn) {
359 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister);
360 } else if (!VRegs.empty()) {
361 MachineFunction &MF = MIRBuilder.getMF();
362 const Function &F = MF.getFunction();
363 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
364
365 MachineRegisterInfo &MRI = MF.getRegInfo();
366 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
367 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
368 auto &DL = F.getParent()->getDataLayout();
369 LLVMContext &Ctx = Val->getType()->getContext();
370
371 SmallVector<EVT, 4> SplitEVTs;
372 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
373 assert(VRegs.size() == SplitEVTs.size() &&(static_cast <bool> (VRegs.size() == SplitEVTs.size() &&
"For each split Type there should be exactly one VReg.") ? void
(0) : __assert_fail ("VRegs.size() == SplitEVTs.size() && \"For each split Type there should be exactly one VReg.\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 374
, __extension__ __PRETTY_FUNCTION__))
374 "For each split Type there should be exactly one VReg.")(static_cast <bool> (VRegs.size() == SplitEVTs.size() &&
"For each split Type there should be exactly one VReg.") ? void
(0) : __assert_fail ("VRegs.size() == SplitEVTs.size() && \"For each split Type there should be exactly one VReg.\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 374
, __extension__ __PRETTY_FUNCTION__))
;
375
376 SmallVector<ArgInfo, 8> SplitArgs;
377 CallingConv::ID CC = F.getCallingConv();
378
379 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
380 Register CurVReg = VRegs[i];
381 ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx), 0};
382 setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
383
384 // i1 is a special case because SDAG i1 true is naturally zero extended
385 // when widened using ANYEXT. We need to do it explicitly here.
386 auto &Flags = CurArgInfo.Flags[0];
387 if (MRI.getType(CurVReg).getSizeInBits() == 1 && !Flags.isSExt() &&
388 !Flags.isZExt()) {
389 CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0);
390 } else if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) ==
391 1) {
392 // Some types will need extending as specified by the CC.
393 MVT NewVT = TLI.getRegisterTypeForCallingConv(Ctx, CC, SplitEVTs[i]);
394 if (EVT(NewVT) != SplitEVTs[i]) {
395 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
396 if (F.getAttributes().hasRetAttr(Attribute::SExt))
397 ExtendOp = TargetOpcode::G_SEXT;
398 else if (F.getAttributes().hasRetAttr(Attribute::ZExt))
399 ExtendOp = TargetOpcode::G_ZEXT;
400
401 LLT NewLLT(NewVT);
402 LLT OldLLT(MVT::getVT(CurArgInfo.Ty));
403 CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Ctx);
404 // Instead of an extend, we might have a vector type which needs
405 // padding with more elements, e.g. <2 x half> -> <4 x half>.
406 if (NewVT.isVector()) {
407 if (OldLLT.isVector()) {
408 if (NewLLT.getNumElements() > OldLLT.getNumElements()) {
409 // We don't handle VA types which are not exactly twice the
410 // size, but can easily be done in future.
411 if (NewLLT.getNumElements() != OldLLT.getNumElements() * 2) {
412 LLVM_DEBUG(dbgs() << "Outgoing vector ret has too many elts")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "Outgoing vector ret has too many elts"
; } } while (false)
;
413 return false;
414 }
415 auto Undef = MIRBuilder.buildUndef({OldLLT});
416 CurVReg =
417 MIRBuilder.buildMergeLikeInstr({NewLLT}, {CurVReg, Undef})
418 .getReg(0);
419 } else {
420 // Just do a vector extend.
421 CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
422 .getReg(0);
423 }
424 } else if (NewLLT.getNumElements() == 2) {
425 // We need to pad a <1 x S> type to <2 x S>. Since we don't have
426 // <1 x S> vector types in GISel we use a build_vector instead
427 // of a vector merge/concat.
428 auto Undef = MIRBuilder.buildUndef({OldLLT});
429 CurVReg =
430 MIRBuilder
431 .buildBuildVector({NewLLT}, {CurVReg, Undef.getReg(0)})
432 .getReg(0);
433 } else {
434 LLVM_DEBUG(dbgs() << "Could not handle ret ty\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "Could not handle ret ty\n"
; } } while (false)
;
435 return false;
436 }
437 } else {
438 // If the split EVT was a <1 x T> vector, and NewVT is T, then we
439 // don't have to do anything since we don't distinguish between the
440 // two.
441 if (NewLLT != MRI.getType(CurVReg)) {
442 // A scalar extend.
443 CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
444 .getReg(0);
445 }
446 }
447 }
448 }
449 if (CurVReg != CurArgInfo.Regs[0]) {
450 CurArgInfo.Regs[0] = CurVReg;
451 // Reset the arg flags after modifying CurVReg.
452 setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
453 }
454 splitToValueTypes(CurArgInfo, SplitArgs, DL, CC);
455 }
456
457 AArch64OutgoingValueAssigner Assigner(AssignFn, AssignFn, Subtarget,
458 /*IsReturn*/ true);
459 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB);
460 Success = determineAndHandleAssignments(Handler, Assigner, SplitArgs,
461 MIRBuilder, CC, F.isVarArg());
462 }
463
464 if (SwiftErrorVReg) {
465 MIB.addUse(AArch64::X21, RegState::Implicit);
466 MIRBuilder.buildCopy(AArch64::X21, SwiftErrorVReg);
467 }
468
469 MIRBuilder.insertInstr(MIB);
470 return Success;
471}
472
473bool AArch64CallLowering::canLowerReturn(MachineFunction &MF,
474 CallingConv::ID CallConv,
475 SmallVectorImpl<BaseArgInfo> &Outs,
476 bool IsVarArg) const {
477 SmallVector<CCValAssign, 16> ArgLocs;
478 const auto &TLI = *getTLI<AArch64TargetLowering>();
479 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
480 MF.getFunction().getContext());
481
482 return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv));
483}
484
485/// Helper function to compute forwarded registers for musttail calls. Computes
486/// the forwarded registers, sets MBB liveness, and emits COPY instructions that
487/// can be used to save + restore registers later.
488static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
489 CCAssignFn *AssignFn) {
490 MachineBasicBlock &MBB = MIRBuilder.getMBB();
491 MachineFunction &MF = MIRBuilder.getMF();
492 MachineFrameInfo &MFI = MF.getFrameInfo();
493
494 if (!MFI.hasMustTailInVarArgFunc())
495 return;
496
497 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
498 const Function &F = MF.getFunction();
499 assert(F.isVarArg() && "Expected F to be vararg?")(static_cast <bool> (F.isVarArg() && "Expected F to be vararg?"
) ? void (0) : __assert_fail ("F.isVarArg() && \"Expected F to be vararg?\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 499
, __extension__ __PRETTY_FUNCTION__))
;
500
501 // Compute the set of forwarded registers. The rest are scratch.
502 SmallVector<CCValAssign, 16> ArgLocs;
503 CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
504 F.getContext());
505 SmallVector<MVT, 2> RegParmTypes;
506 RegParmTypes.push_back(MVT::i64);
507 RegParmTypes.push_back(MVT::f128);
508
509 // Later on, we can use this vector to restore the registers if necessary.
510 SmallVectorImpl<ForwardedRegister> &Forwards =
511 FuncInfo->getForwardedMustTailRegParms();
512 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, AssignFn);
513
514 // Conservatively forward X8, since it might be used for an aggregate
515 // return.
516 if (!CCInfo.isAllocated(AArch64::X8)) {
517 Register X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
518 Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
519 }
520
521 // Add the forwards to the MachineBasicBlock and MachineFunction.
522 for (const auto &F : Forwards) {
523 MBB.addLiveIn(F.PReg);
524 MIRBuilder.buildCopy(Register(F.VReg), Register(F.PReg));
525 }
526}
527
528bool AArch64CallLowering::fallBackToDAGISel(const MachineFunction &MF) const {
529 auto &F = MF.getFunction();
530 if (F.getReturnType()->isScalableTy() ||
531 llvm::any_of(F.args(), [](const Argument &A) {
532 return A.getType()->isScalableTy();
533 }))
534 return true;
535 const auto &ST = MF.getSubtarget<AArch64Subtarget>();
536 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
537 LLVM_DEBUG(dbgs() << "Falling back to SDAG because we don't support no-NEON\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "Falling back to SDAG because we don't support no-NEON\n"
; } } while (false)
;
538 return true;
539 }
540
541 SMEAttrs Attrs(F);
542 if (Attrs.hasNewZAInterface() ||
543 (!Attrs.hasStreamingInterface() && Attrs.hasStreamingBody()))
544 return true;
545
546 return false;
547}
548
549void AArch64CallLowering::saveVarArgRegisters(
550 MachineIRBuilder &MIRBuilder, CallLowering::IncomingValueHandler &Handler,
551 CCState &CCInfo) const {
552 auto GPRArgRegs = AArch64::getGPRArgRegs();
553 auto FPRArgRegs = AArch64::getFPRArgRegs();
554
555 MachineFunction &MF = MIRBuilder.getMF();
556 MachineRegisterInfo &MRI = MF.getRegInfo();
557 MachineFrameInfo &MFI = MF.getFrameInfo();
558 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
559 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
560 bool IsWin64CC =
561 Subtarget.isCallingConvWin64(CCInfo.getCallingConv());
562 const LLT p0 = LLT::pointer(0, 64);
563 const LLT s64 = LLT::scalar(64);
564
565 unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
566 unsigned NumVariadicGPRArgRegs = GPRArgRegs.size() - FirstVariadicGPR + 1;
567
568 unsigned GPRSaveSize = 8 * (GPRArgRegs.size() - FirstVariadicGPR);
569 int GPRIdx = 0;
570 if (GPRSaveSize != 0) {
571 if (IsWin64CC) {
572 GPRIdx = MFI.CreateFixedObject(GPRSaveSize,
573 -static_cast<int>(GPRSaveSize), false);
574 } else
575 GPRIdx = MFI.CreateStackObject(GPRSaveSize, Align(8), false);
576
577 auto FIN = MIRBuilder.buildFrameIndex(p0, GPRIdx);
578 auto Offset =
579 MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 8);
580
581 for (unsigned i = FirstVariadicGPR; i < GPRArgRegs.size(); ++i) {
582 Register Val = MRI.createGenericVirtualRegister(s64);
583 Handler.assignValueToReg(
584 Val, GPRArgRegs[i],
585 CCValAssign::getReg(i + MF.getFunction().getNumOperands(), MVT::i64,
586 GPRArgRegs[i], MVT::i64, CCValAssign::Full));
587 auto MPO = IsWin64CC ? MachinePointerInfo::getFixedStack(
588 MF, GPRIdx, (i - FirstVariadicGPR) * 8)
589 : MachinePointerInfo::getStack(MF, i * 8);
590 MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
591
592 FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
593 FIN.getReg(0), Offset);
594 }
595 }
596 FuncInfo->setVarArgsGPRIndex(GPRIdx);
597 FuncInfo->setVarArgsGPRSize(GPRSaveSize);
598
599 if (Subtarget.hasFPARMv8() && !IsWin64CC) {
600 unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
601
602 unsigned FPRSaveSize = 16 * (FPRArgRegs.size() - FirstVariadicFPR);
603 int FPRIdx = 0;
604 if (FPRSaveSize != 0) {
605 FPRIdx = MFI.CreateStackObject(FPRSaveSize, Align(16), false);
606
607 auto FIN = MIRBuilder.buildFrameIndex(p0, FPRIdx);
608 auto Offset =
609 MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 16);
610
611 for (unsigned i = FirstVariadicFPR; i < FPRArgRegs.size(); ++i) {
612 Register Val = MRI.createGenericVirtualRegister(LLT::scalar(128));
613 Handler.assignValueToReg(
614 Val, FPRArgRegs[i],
615 CCValAssign::getReg(
616 i + MF.getFunction().getNumOperands() + NumVariadicGPRArgRegs,
617 MVT::f128, FPRArgRegs[i], MVT::f128, CCValAssign::Full));
618
619 auto MPO = MachinePointerInfo::getStack(MF, i * 16);
620 MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
621
622 FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
623 FIN.getReg(0), Offset);
624 }
625 }
626 FuncInfo->setVarArgsFPRIndex(FPRIdx);
627 FuncInfo->setVarArgsFPRSize(FPRSaveSize);
628 }
629}
630
631bool AArch64CallLowering::lowerFormalArguments(
632 MachineIRBuilder &MIRBuilder, const Function &F,
633 ArrayRef<ArrayRef<Register>> VRegs, FunctionLoweringInfo &FLI) const {
634 MachineFunction &MF = MIRBuilder.getMF();
635 MachineBasicBlock &MBB = MIRBuilder.getMBB();
636 MachineRegisterInfo &MRI = MF.getRegInfo();
637 auto &DL = F.getParent()->getDataLayout();
638 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
639 // TODO: Support Arm64EC
640 bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv()) && !Subtarget.isWindowsArm64EC();
641
642 SmallVector<ArgInfo, 8> SplitArgs;
643 SmallVector<std::pair<Register, Register>> BoolArgs;
644
645 // Insert the hidden sret parameter if the return value won't fit in the
646 // return registers.
647 if (!FLI.CanLowerReturn)
648 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL);
649
650 unsigned i = 0;
651 for (auto &Arg : F.args()) {
652 if (DL.getTypeStoreSize(Arg.getType()).isZero())
653 continue;
654
655 ArgInfo OrigArg{VRegs[i], Arg, i};
656 setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
657
658 // i1 arguments are zero-extended to i8 by the caller. Emit a
659 // hint to reflect this.
660 if (OrigArg.Ty->isIntegerTy(1)) {
661 assert(OrigArg.Regs.size() == 1 &&(static_cast <bool> (OrigArg.Regs.size() == 1 &&
MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
"Unexpected registers used for i1 arg") ? void (0) : __assert_fail
("OrigArg.Regs.size() == 1 && MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 && \"Unexpected registers used for i1 arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 663
, __extension__ __PRETTY_FUNCTION__))
662 MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&(static_cast <bool> (OrigArg.Regs.size() == 1 &&
MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
"Unexpected registers used for i1 arg") ? void (0) : __assert_fail
("OrigArg.Regs.size() == 1 && MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 && \"Unexpected registers used for i1 arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 663
, __extension__ __PRETTY_FUNCTION__))
663 "Unexpected registers used for i1 arg")(static_cast <bool> (OrigArg.Regs.size() == 1 &&
MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
"Unexpected registers used for i1 arg") ? void (0) : __assert_fail
("OrigArg.Regs.size() == 1 && MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 && \"Unexpected registers used for i1 arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 663
, __extension__ __PRETTY_FUNCTION__))
;
664
665 auto &Flags = OrigArg.Flags[0];
666 if (!Flags.isZExt() && !Flags.isSExt()) {
667 // Lower i1 argument as i8, and insert AssertZExt + Trunc later.
668 Register OrigReg = OrigArg.Regs[0];
669 Register WideReg = MRI.createGenericVirtualRegister(LLT::scalar(8));
670 OrigArg.Regs[0] = WideReg;
671 BoolArgs.push_back({OrigReg, WideReg});
672 }
673 }
674
675 if (Arg.hasAttribute(Attribute::SwiftAsync))
676 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
677
678 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());
679 ++i;
680 }
681
682 if (!MBB.empty())
683 MIRBuilder.setInstr(*MBB.begin());
684
685 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
686 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(F.getCallingConv(), IsWin64 && F.isVarArg());
687
688 AArch64IncomingValueAssigner Assigner(AssignFn, AssignFn);
689 FormalArgHandler Handler(MIRBuilder, MRI);
690 SmallVector<CCValAssign, 16> ArgLocs;
691 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
692 if (!determineAssignments(Assigner, SplitArgs, CCInfo) ||
693 !handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, MIRBuilder))
694 return false;
695
696 if (!BoolArgs.empty()) {
697 for (auto &KV : BoolArgs) {
698 Register OrigReg = KV.first;
699 Register WideReg = KV.second;
700 LLT WideTy = MRI.getType(WideReg);
701 assert(MRI.getType(OrigReg).getScalarSizeInBits() == 1 &&(static_cast <bool> (MRI.getType(OrigReg).getScalarSizeInBits
() == 1 && "Unexpected bit size of a bool arg") ? void
(0) : __assert_fail ("MRI.getType(OrigReg).getScalarSizeInBits() == 1 && \"Unexpected bit size of a bool arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 702
, __extension__ __PRETTY_FUNCTION__))
702 "Unexpected bit size of a bool arg")(static_cast <bool> (MRI.getType(OrigReg).getScalarSizeInBits
() == 1 && "Unexpected bit size of a bool arg") ? void
(0) : __assert_fail ("MRI.getType(OrigReg).getScalarSizeInBits() == 1 && \"Unexpected bit size of a bool arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 702
, __extension__ __PRETTY_FUNCTION__))
;
703 MIRBuilder.buildTrunc(
704 OrigReg, MIRBuilder.buildAssertZExt(WideTy, WideReg, 1).getReg(0));
705 }
706 }
707
708 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
709 uint64_t StackOffset = Assigner.StackOffset;
710 if (F.isVarArg()) {
711 if ((!Subtarget.isTargetDarwin() && !Subtarget.isWindowsArm64EC()) || IsWin64) {
712 // The AAPCS variadic function ABI is identical to the non-variadic
713 // one. As a result there may be more arguments in registers and we should
714 // save them for future reference.
715 // Win64 variadic functions also pass arguments in registers, but all
716 // float arguments are passed in integer registers.
717 saveVarArgRegisters(MIRBuilder, Handler, CCInfo);
718 } else if (Subtarget.isWindowsArm64EC()) {
719 return false;
720 }
721
722 // We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
723 StackOffset =
724 alignTo(Assigner.StackOffset, Subtarget.isTargetILP32() ? 4 : 8);
725
726 auto &MFI = MIRBuilder.getMF().getFrameInfo();
727 FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
728 }
729
730 if (doesCalleeRestoreStack(F.getCallingConv(),
731 MF.getTarget().Options.GuaranteedTailCallOpt)) {
732 // We have a non-standard ABI, so why not make full use of the stack that
733 // we're going to pop? It must be aligned to 16 B in any case.
734 StackOffset = alignTo(StackOffset, 16);
735
736 // If we're expected to restore the stack (e.g. fastcc), then we'll be
737 // adding a multiple of 16.
738 FuncInfo->setArgumentStackToRestore(StackOffset);
739
740 // Our own callers will guarantee that the space is free by giving an
741 // aligned value to CALLSEQ_START.
742 }
743
744 // When we tail call, we need to check if the callee's arguments
745 // will fit on the caller's stack. So, whenever we lower formal arguments,
746 // we should keep track of this information, since we might lower a tail call
747 // in this function later.
748 FuncInfo->setBytesInStackArgArea(StackOffset);
749
750 if (Subtarget.hasCustomCallingConv())
751 Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
752
753 handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
754
755 // Move back to the end of the basic block.
756 MIRBuilder.setMBB(MBB);
757
758 return true;
759}
760
761/// Return true if the calling convention is one that we can guarantee TCO for.
762static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
763 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
764 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
765}
766
767/// Return true if we might ever do TCO for calls with this calling convention.
768static bool mayTailCallThisCC(CallingConv::ID CC) {
769 switch (CC) {
770 case CallingConv::C:
771 case CallingConv::PreserveMost:
772 case CallingConv::Swift:
773 case CallingConv::SwiftTail:
774 case CallingConv::Tail:
775 case CallingConv::Fast:
776 return true;
777 default:
778 return false;
779 }
780}
781
782/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
783/// CC.
784static std::pair<CCAssignFn *, CCAssignFn *>
785getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI) {
786 return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
787}
788
789bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
790 CallLoweringInfo &Info, MachineFunction &MF,
791 SmallVectorImpl<ArgInfo> &InArgs) const {
792 const Function &CallerF = MF.getFunction();
793 CallingConv::ID CalleeCC = Info.CallConv;
794 CallingConv::ID CallerCC = CallerF.getCallingConv();
795
796 // If the calling conventions match, then everything must be the same.
797 if (CalleeCC == CallerCC)
798 return true;
799
800 // Check if the caller and callee will handle arguments in the same way.
801 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
802 CCAssignFn *CalleeAssignFnFixed;
803 CCAssignFn *CalleeAssignFnVarArg;
804 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
805 getAssignFnsForCC(CalleeCC, TLI);
806
807 CCAssignFn *CallerAssignFnFixed;
808 CCAssignFn *CallerAssignFnVarArg;
809 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
810 getAssignFnsForCC(CallerCC, TLI);
811
812 AArch64IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
813 CalleeAssignFnVarArg);
814 AArch64IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
815 CallerAssignFnVarArg);
816
817 if (!resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner))
818 return false;
819
820 // Make sure that the caller and callee preserve all of the same registers.
821 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
822 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
823 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
824 if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv()) {
825 TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
826 TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
827 }
828
829 return TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved);
830}
831
832bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
833 CallLoweringInfo &Info, MachineFunction &MF,
834 SmallVectorImpl<ArgInfo> &OutArgs) const {
835 // If there are no outgoing arguments, then we are done.
836 if (OutArgs.empty())
837 return true;
838
839 const Function &CallerF = MF.getFunction();
840 LLVMContext &Ctx = CallerF.getContext();
841 CallingConv::ID CalleeCC = Info.CallConv;
842 CallingConv::ID CallerCC = CallerF.getCallingConv();
843 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
844 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
845
846 CCAssignFn *AssignFnFixed;
847 CCAssignFn *AssignFnVarArg;
848 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
849
850 // We have outgoing arguments. Make sure that we can tail call with them.
851 SmallVector<CCValAssign, 16> OutLocs;
852 CCState OutInfo(CalleeCC, false, MF, OutLocs, Ctx);
853
854 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
855 Subtarget, /*IsReturn*/ false);
856 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo)) {
857 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Could not analyze call operands.\n"
; } } while (false)
;
858 return false;
859 }
860
861 // Make sure that they can fit on the caller's stack.
862 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
863 if (OutInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) {
864 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot fit call operands on caller's stack.\n"
; } } while (false)
;
865 return false;
866 }
867
868 // Verify that the parameters in callee-saved registers match.
869 // TODO: Port this over to CallLowering as general code once swiftself is
870 // supported.
871 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
872 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
873 MachineRegisterInfo &MRI = MF.getRegInfo();
874
875 if (Info.IsVarArg) {
876 // Be conservative and disallow variadic memory operands to match SDAG's
877 // behaviour.
878 // FIXME: If the caller's calling convention is C, then we can
879 // potentially use its argument area. However, for cases like fastcc,
880 // we can't do anything.
881 for (unsigned i = 0; i < OutLocs.size(); ++i) {
882 auto &ArgLoc = OutLocs[i];
883 if (ArgLoc.isRegLoc())
884 continue;
885
886 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot tail call vararg function with stack arguments\n"
; } } while (false)
887 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot tail call vararg function with stack arguments\n"
; } } while (false)
888 << "... Cannot tail call vararg function with stack arguments\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot tail call vararg function with stack arguments\n"
; } } while (false)
;
889 return false;
890 }
891 }
892
893 return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);
894}
895
896bool AArch64CallLowering::isEligibleForTailCallOptimization(
897 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
898 SmallVectorImpl<ArgInfo> &InArgs,
899 SmallVectorImpl<ArgInfo> &OutArgs) const {
900
901 // Must pass all target-independent checks in order to tail call optimize.
902 if (!Info.IsTailCall)
6
Assuming field 'IsTailCall' is true
7
Taking false branch
903 return false;
904
905 CallingConv::ID CalleeCC = Info.CallConv;
906 MachineFunction &MF = MIRBuilder.getMF();
907 const Function &CallerF = MF.getFunction();
908
909 LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "Attempting to lower call as tail call\n"
; } } while (false)
;
8
Assuming 'DebugFlag' is false
9
Loop condition is false. Exiting loop
910
911 if (Info.SwiftErrorVReg) {
10
Assuming the condition is false
11
Taking false branch
912 // TODO: We should handle this.
913 // Note that this is also handled by the check for no outgoing arguments.
914 // Proactively disabling this though, because the swifterror handling in
915 // lowerCall inserts a COPY *after* the location of the call.
916 LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot handle tail calls with swifterror yet.\n"
; } } while (false)
;
917 return false;
918 }
919
920 if (!mayTailCallThisCC(CalleeCC)) {
921 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Calling convention cannot be tail called.\n"
; } } while (false)
;
12
Taking true branch
13
Loop condition is false. Exiting loop
922 return false;
14
Returning zero, which participates in a condition later
923 }
924
925 // Byval parameters hand the function a pointer directly into the stack area
926 // we want to reuse during a tail call. Working around this *is* possible (see
927 // X86).
928 //
929 // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
930 // it?
931 //
932 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
933 // In this case, it is necessary to save/restore X0 in the callee. Tail
934 // call opt interferes with this. So we disable tail call opt when the
935 // caller has an argument with "inreg" attribute.
936 //
937 // FIXME: Check whether the callee also has an "inreg" argument.
938 //
939 // When the caller has a swifterror argument, we don't want to tail call
940 // because would have to move into the swifterror register before the
941 // tail call.
942 if (any_of(CallerF.args(), [](const Argument &A) {
943 return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
944 })) {
945 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot tail call from callers with byval, "
"inreg, or swifterror arguments\n"; } } while (false)
946 "inreg, or swifterror arguments\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot tail call from callers with byval, "
"inreg, or swifterror arguments\n"; } } while (false)
;
947 return false;
948 }
949
950 // Externally-defined functions with weak linkage should not be
951 // tail-called on AArch64 when the OS does not support dynamic
952 // pre-emption of symbols, as the AAELF spec requires normal calls
953 // to undefined weak functions to be replaced with a NOP or jump to the
954 // next instruction. The behaviour of branch instructions in this
955 // situation (as used for tail calls) is implementation-defined, so we
956 // cannot rely on the linker replacing the tail call with a return.
957 if (Info.Callee.isGlobal()) {
958 const GlobalValue *GV = Info.Callee.getGlobal();
959 const Triple &TT = MF.getTarget().getTargetTriple();
960 if (GV->hasExternalWeakLinkage() &&
961 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
962 TT.isOSBinFormatMachO())) {
963 LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot tail call externally-defined function "
"with weak linkage for this OS.\n"; } } while (false)
964 "with weak linkage for this OS.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Cannot tail call externally-defined function "
"with weak linkage for this OS.\n"; } } while (false)
;
965 return false;
966 }
967 }
968
969 // If we have -tailcallopt, then we're done.
970 if (canGuaranteeTCO(CalleeCC, MF.getTarget().Options.GuaranteedTailCallOpt))
971 return CalleeCC == CallerF.getCallingConv();
972
973 // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
974 // Try to find cases where we can do that.
975
976 // I want anyone implementing a new calling convention to think long and hard
977 // about this assert.
978 assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&(static_cast <bool> ((!Info.IsVarArg || CalleeCC == CallingConv
::C) && "Unexpected variadic calling convention") ? void
(0) : __assert_fail ("(!Info.IsVarArg || CalleeCC == CallingConv::C) && \"Unexpected variadic calling convention\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 979
, __extension__ __PRETTY_FUNCTION__))
979 "Unexpected variadic calling convention")(static_cast <bool> ((!Info.IsVarArg || CalleeCC == CallingConv
::C) && "Unexpected variadic calling convention") ? void
(0) : __assert_fail ("(!Info.IsVarArg || CalleeCC == CallingConv::C) && \"Unexpected variadic calling convention\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 979
, __extension__ __PRETTY_FUNCTION__))
;
980
981 // Verify that the incoming and outgoing arguments from the callee are
982 // safe to tail call.
983 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
984 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
985 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
986 << "... Caller and callee have incompatible calling conventions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Caller and callee have incompatible calling conventions.\n"
; } } while (false)
;
987 return false;
988 }
989
990 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
991 return false;
992
993 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Call is eligible for tail call optimization.\n"
; } } while (false)
994 dbgs() << "... Call is eligible for tail call optimization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "... Call is eligible for tail call optimization.\n"
; } } while (false)
;
995 return true;
996}
997
998static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
999 bool IsTailCall) {
1000 if (!IsTailCall)
1001 return IsIndirect ? getBLRCallOpcode(CallerF) : (unsigned)AArch64::BL;
1002
1003 if (!IsIndirect)
1004 return AArch64::TCRETURNdi;
1005
1006 // When BTI is enabled, we need to use TCRETURNriBTI to make sure that we use
1007 // x16 or x17.
1008 if (CallerF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
1009 return AArch64::TCRETURNriBTI;
1010
1011 return AArch64::TCRETURNri;
1012}
1013
1014static const uint32_t *
1015getMaskForArgs(SmallVectorImpl<AArch64CallLowering::ArgInfo> &OutArgs,
1016 AArch64CallLowering::CallLoweringInfo &Info,
1017 const AArch64RegisterInfo &TRI, MachineFunction &MF) {
1018 const uint32_t *Mask;
1019 if (!OutArgs.empty() && OutArgs[0].Flags[0].isReturned()) {
1020 // For 'this' returns, use the X0-preserving mask if applicable
1021 Mask = TRI.getThisReturnPreservedMask(MF, Info.CallConv);
1022 if (!Mask) {
1023 OutArgs[0].Flags[0].setReturned(false);
1024 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1025 }
1026 } else {
1027 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1028 }
1029 return Mask;
1030}
1031
1032bool AArch64CallLowering::lowerTailCall(
1033 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1034 SmallVectorImpl<ArgInfo> &OutArgs) const {
1035 MachineFunction &MF = MIRBuilder.getMF();
1036 const Function &F = MF.getFunction();
1037 MachineRegisterInfo &MRI = MF.getRegInfo();
1038 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1039 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1040
1041 // True when we're tail calling, but without -tailcallopt.
1042 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt &&
1043 Info.CallConv != CallingConv::Tail &&
1044 Info.CallConv != CallingConv::SwiftTail;
1045
1046 // TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
1047 // register class. Until we can do that, we should fall back here.
1048 if (MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement()) {
1049 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n"
; } } while (false)
1050 dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n"
; } } while (false)
;
1051 return false;
1052 }
1053
1054 // Find out which ABI gets to decide where things go.
1055 CallingConv::ID CalleeCC = Info.CallConv;
1056 CCAssignFn *AssignFnFixed;
1057 CCAssignFn *AssignFnVarArg;
1058 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
1059
1060 MachineInstrBuilder CallSeqStart;
1061 if (!IsSibCall)
1062 CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1063
1064 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true);
1065 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1066 MIB.add(Info.Callee);
1067
1068 // Byte offset for the tail call. When we are sibcalling, this will always
1069 // be 0.
1070 MIB.addImm(0);
1071
1072 // Tell the call which registers are clobbered.
1073 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1074 auto TRI = Subtarget.getRegisterInfo();
1075 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1076 if (Subtarget.hasCustomCallingConv())
1077 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1078 MIB.addRegMask(Mask);
1079
1080 if (Info.CFIType)
1081 MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1082
1083 if (TRI->isAnyArgRegReserved(MF))
1084 TRI->emitReservedArgRegCallError(MF);
1085
1086 // FPDiff is the byte offset of the call's argument area from the callee's.
1087 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1088 // by this amount for a tail call. In a sibling call it must be 0 because the
1089 // caller will deallocate the entire stack and the callee still expects its
1090 // arguments to begin at SP+0.
1091 int FPDiff = 0;
1092
1093 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1094 // by -tailcallopt. For sibcalls, the memory operands for the call are
1095 // already available in the caller's incoming argument space.
1096 unsigned NumBytes = 0;
1097 if (!IsSibCall) {
1098 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1099 // before handling assignments, because FPDiff must be known for memory
1100 // arguments.
1101 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1102 SmallVector<CCValAssign, 16> OutLocs;
1103 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1104
1105 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
1106 Subtarget, /*IsReturn*/ false);
1107 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo))
1108 return false;
1109
1110 // The callee will pop the argument stack as a tail call. Thus, we must
1111 // keep it 16-byte aligned.
1112 NumBytes = alignTo(OutInfo.getNextStackOffset(), 16);
1113
1114 // FPDiff will be negative if this tail call requires more space than we
1115 // would automatically have in our incoming argument space. Positive if we
1116 // actually shrink the stack.
1117 FPDiff = NumReusableBytes - NumBytes;
1118
1119 // Update the required reserved area if this is the tail call requiring the
1120 // most argument stack space.
1121 if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff)
1122 FuncInfo->setTailCallReservedStack(-FPDiff);
1123
1124 // The stack pointer must be 16-byte aligned at all times it's used for a
1125 // memory operation, which in practice means at *all* times and in
1126 // particular across call boundaries. Therefore our own arguments started at
1127 // a 16-byte aligned SP and the delta applied for the tail call should
1128 // satisfy the same constraint.
1129 assert(FPDiff % 16 == 0 && "unaligned stack on tail call")(static_cast <bool> (FPDiff % 16 == 0 && "unaligned stack on tail call"
) ? void (0) : __assert_fail ("FPDiff % 16 == 0 && \"unaligned stack on tail call\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 1129
, __extension__ __PRETTY_FUNCTION__))
;
1130 }
1131
1132 const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
1133
1134 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1135 Subtarget, /*IsReturn*/ false);
1136
1137 // Do the actual argument marshalling.
1138 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB,
1139 /*IsTailCall*/ true, FPDiff);
1140 if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1141 CalleeCC, Info.IsVarArg))
1142 return false;
1143
1144 Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1145
1146 if (Info.IsVarArg && Info.IsMustTailCall) {
1147 // Now we know what's being passed to the function. Add uses to the call for
1148 // the forwarded registers that we *aren't* passing as parameters. This will
1149 // preserve the copies we build earlier.
1150 for (const auto &F : Forwards) {
1151 Register ForwardedReg = F.PReg;
1152 // If the register is already passed, or aliases a register which is
1153 // already being passed, then skip it.
1154 if (any_of(MIB->uses(), [&ForwardedReg, &TRI](const MachineOperand &Use) {
1155 if (!Use.isReg())
1156 return false;
1157 return TRI->regsOverlap(Use.getReg(), ForwardedReg);
1158 }))
1159 continue;
1160
1161 // We aren't passing it already, so we should add it to the call.
1162 MIRBuilder.buildCopy(ForwardedReg, Register(F.VReg));
1163 MIB.addReg(ForwardedReg, RegState::Implicit);
1164 }
1165 }
1166
1167 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1168 // sequence start and end here.
1169 if (!IsSibCall) {
1170 MIB->getOperand(1).setImm(FPDiff);
1171 CallSeqStart.addImm(0).addImm(0);
1172 // End the call sequence *before* emitting the call. Normally, we would
1173 // tidy the frame up after the call. However, here, we've laid out the
1174 // parameters so that when SP is reset, they will be in the correct
1175 // location.
1176 MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP).addImm(0).addImm(0);
1177 }
1178
1179 // Now we can add the actual call instruction to the correct basic block.
1180 MIRBuilder.insertInstr(MIB);
1181
1182 // If Callee is a reg, since it is used by a target specific instruction,
1183 // it must have a register class matching the constraint of that instruction.
1184 if (MIB->getOperand(0).isReg())
1185 constrainOperandRegClass(MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
1186 *MF.getSubtarget().getRegBankInfo(), *MIB,
1187 MIB->getDesc(), MIB->getOperand(0), 0);
1188
1189 MF.getFrameInfo().setHasTailCall();
1190 Info.LoweredTailCall = true;
1191 return true;
1192}
1193
1194bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
1195 CallLoweringInfo &Info) const {
1196 MachineFunction &MF = MIRBuilder.getMF();
1197 const Function &F = MF.getFunction();
1198 MachineRegisterInfo &MRI = MF.getRegInfo();
1199 auto &DL = F.getParent()->getDataLayout();
1200 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1201 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1202
1203 // Arm64EC has extra requirements for varargs calls; bail out for now.
1204 if (Info.IsVarArg && Subtarget.isWindowsArm64EC())
1
Assuming field 'IsVarArg' is false
1205 return false;
1206
1207 SmallVector<ArgInfo, 8> OutArgs;
1208 for (auto &OrigArg : Info.OrigArgs) {
2
Value assigned to field 'CB'
3
Assuming '__begin1' is equal to '__end1'
1209 splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);
1210 // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
1211 auto &Flags = OrigArg.Flags[0];
1212 if (OrigArg.Ty->isIntegerTy(1) && !Flags.isSExt() && !Flags.isZExt()) {
1213 ArgInfo &OutArg = OutArgs.back();
1214 assert(OutArg.Regs.size() == 1 &&(static_cast <bool> (OutArg.Regs.size() == 1 &&
MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 && "Unexpected registers used for i1 arg"
) ? void (0) : __assert_fail ("OutArg.Regs.size() == 1 && MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 && \"Unexpected registers used for i1 arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 1216
, __extension__ __PRETTY_FUNCTION__))
1215 MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 &&(static_cast <bool> (OutArg.Regs.size() == 1 &&
MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 && "Unexpected registers used for i1 arg"
) ? void (0) : __assert_fail ("OutArg.Regs.size() == 1 && MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 && \"Unexpected registers used for i1 arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 1216
, __extension__ __PRETTY_FUNCTION__))
1216 "Unexpected registers used for i1 arg")(static_cast <bool> (OutArg.Regs.size() == 1 &&
MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 && "Unexpected registers used for i1 arg"
) ? void (0) : __assert_fail ("OutArg.Regs.size() == 1 && MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 && \"Unexpected registers used for i1 arg\""
, "llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp", 1216
, __extension__ __PRETTY_FUNCTION__))
;
1217
1218 // We cannot use a ZExt ArgInfo flag here, because it will
1219 // zero-extend the argument to i32 instead of just i8.
1220 OutArg.Regs[0] =
1221 MIRBuilder.buildZExt(LLT::scalar(8), OutArg.Regs[0]).getReg(0);
1222 LLVMContext &Ctx = MF.getFunction().getContext();
1223 OutArg.Ty = Type::getInt8Ty(Ctx);
1224 }
1225 }
1226
1227 SmallVector<ArgInfo, 8> InArgs;
1228 if (!Info.OrigRet.Ty->isVoidTy())
4
Taking true branch
1229 splitToValueTypes(Info.OrigRet, InArgs, DL, Info.CallConv);
1230
1231 // If we can lower as a tail call, do that instead.
1232 bool CanTailCallOpt =
1233 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
5
Calling 'AArch64CallLowering::isEligibleForTailCallOptimization'
15
Returning from 'AArch64CallLowering::isEligibleForTailCallOptimization'
1234
1235 // We must emit a tail call if we have musttail.
1236 if (Info.IsMustTailCall && !CanTailCallOpt) {
16
Assuming field 'IsMustTailCall' is false
1237 // There are types of incoming/outgoing arguments we can't handle yet, so
1238 // it doesn't make sense to actually die here like in ISelLowering. Instead,
1239 // fall back to SelectionDAG and let it try to handle this.
1240 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-call-lowering")) { dbgs() << "Failed to lower musttail call as tail call\n"
; } } while (false)
;
1241 return false;
1242 }
1243
1244 Info.IsTailCall = CanTailCallOpt;
1245 if (CanTailCallOpt
16.1
'CanTailCallOpt' is false
16.1
'CanTailCallOpt' is false
)
17
Taking false branch
1246 return lowerTailCall(MIRBuilder, Info, OutArgs);
1247
1248 // Find out which ABI gets to decide where things go.
1249 CCAssignFn *AssignFnFixed;
1250 CCAssignFn *AssignFnVarArg;
1251 std::tie(AssignFnFixed, AssignFnVarArg) =
1252 getAssignFnsForCC(Info.CallConv, TLI);
1253
1254 MachineInstrBuilder CallSeqStart;
1255 CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1256
1257 // Create a temporarily-floating call instruction so we can add the implicit
1258 // uses of arg registers.
1259
1260 unsigned Opc = 0;
1261 // Calls with operand bundle "clang.arc.attachedcall" are special. They should
1262 // be expanded to the call, directly followed by a special marker sequence and
1263 // a call to an ObjC library function.
1264 if (Info.CB && objcarc::hasAttachedCallOpBundle(Info.CB))
18
Assuming field 'CB' is null
1265 Opc = AArch64::BLR_RVMARKER;
1266 // A call to a returns twice function like setjmp must be followed by a bti
1267 // instruction.
1268 else if (Info.CB
18.1
Field 'CB' is null
18.1
Field 'CB' is null
&& Info.CB->hasFnAttr(Attribute::ReturnsTwice) &&
1269 !Subtarget.noBTIAtReturnTwice() &&
1270 MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
1271 Opc = AArch64::BLR_BTI;
1272 else
1273 Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
1274
1275 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1276 unsigned CalleeOpNo = 0;
1277
1278 if (Opc == AArch64::BLR_RVMARKER) {
19
Assuming 'Opc' is equal to BLR_RVMARKER
20
Taking true branch
1279 // Add a target global address for the retainRV/claimRV runtime function
1280 // just before the call target.
1281 Function *ARCFn = *objcarc::getAttachedARCFunction(Info.CB);
21
Passing null pointer value via 1st parameter 'CB'
22
Calling 'getAttachedARCFunction'
1282 MIB.addGlobalAddress(ARCFn);
1283 ++CalleeOpNo;
1284 } else if (Info.CFIType) {
1285 MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1286 }
1287
1288 MIB.add(Info.Callee);
1289
1290 // Tell the call which registers are clobbered.
1291 const uint32_t *Mask;
1292 const auto *TRI = Subtarget.getRegisterInfo();
1293
1294 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1295 Subtarget, /*IsReturn*/ false);
1296 // Do the actual argument marshalling.
1297 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, /*IsReturn*/ false);
1298 if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1299 Info.CallConv, Info.IsVarArg))
1300 return false;
1301
1302 Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1303
1304 if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
1305 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1306 MIB.addRegMask(Mask);
1307
1308 if (TRI->isAnyArgRegReserved(MF))
1309 TRI->emitReservedArgRegCallError(MF);
1310
1311 // Now we can add the actual call instruction to the correct basic block.
1312 MIRBuilder.insertInstr(MIB);
1313
1314 uint64_t CalleePopBytes =
1315 doesCalleeRestoreStack(Info.CallConv,
1316 MF.getTarget().Options.GuaranteedTailCallOpt)
1317 ? alignTo(Assigner.StackOffset, 16)
1318 : 0;
1319
1320 CallSeqStart.addImm(Assigner.StackOffset).addImm(0);
1321 MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
1322 .addImm(Assigner.StackOffset)
1323 .addImm(CalleePopBytes);
1324
1325 // If Callee is a reg, since it is used by a target specific
1326 // instruction, it must have a register class matching the
1327 // constraint of that instruction.
1328 if (MIB->getOperand(CalleeOpNo).isReg())
1329 constrainOperandRegClass(MF, *TRI, MRI, *Subtarget.getInstrInfo(),
1330 *Subtarget.getRegBankInfo(), *MIB, MIB->getDesc(),
1331 MIB->getOperand(CalleeOpNo), CalleeOpNo);
1332
1333 // Finally we can copy the returned value back into its virtual-register. In
1334 // symmetry with the arguments, the physical register must be an
1335 // implicit-define of the call instruction.
1336 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1337 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv);
1338 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1339 bool UsingReturnedArg =
1340 !OutArgs.empty() && OutArgs[0].Flags[0].isReturned();
1341
1342 AArch64OutgoingValueAssigner Assigner(RetAssignFn, RetAssignFn, Subtarget,
1343 /*IsReturn*/ false);
1344 ReturnedArgCallReturnHandler ReturnedArgHandler(MIRBuilder, MRI, MIB);
1345 if (!determineAndHandleAssignments(
1346 UsingReturnedArg ? ReturnedArgHandler : Handler, Assigner, InArgs,
1347 MIRBuilder, Info.CallConv, Info.IsVarArg,
1348 UsingReturnedArg ? ArrayRef(OutArgs[0].Regs) : std::nullopt))
1349 return false;
1350 }
1351
1352 if (Info.SwiftErrorVReg) {
1353 MIB.addDef(AArch64::X21, RegState::Implicit);
1354 MIRBuilder.buildCopy(Info.SwiftErrorVReg, Register(AArch64::X21));
1355 }
1356
1357 if (!Info.CanLowerReturn) {
1358 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,
1359 Info.DemoteRegister, Info.DemoteStackIndex);
1360 }
1361 return true;
1362}
1363
1364bool AArch64CallLowering::isTypeIsValidForThisReturn(EVT Ty) const {
1365 return Ty.getSizeInBits() == 64;
1366}

/build/source/llvm/include/llvm/Analysis/ObjCARCUtil.h

1//===- ObjCARCUtil.h - ObjC ARC Utility Functions ---------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file defines ARC utility functions which are used by various parts of
10/// the compiler.
11///
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_ANALYSIS_OBJCARCUTIL_H
15#define LLVM_ANALYSIS_OBJCARCUTIL_H
16
17#include "llvm/Analysis/ObjCARCInstKind.h"
18#include "llvm/IR/Function.h"
19#include "llvm/IR/InstrTypes.h"
20#include "llvm/IR/LLVMContext.h"
21
22namespace llvm {
23namespace objcarc {
24
25inline const char *getRVMarkerModuleFlagStr() {
26 return "clang.arc.retainAutoreleasedReturnValueMarker";
27}
28
29inline bool hasAttachedCallOpBundle(const CallBase *CB) {
30 // Ignore the bundle if the return type is void. Global optimization passes
31 // can turn the called function's return type to void. That should happen only
32 // if the call doesn't return and the call to @llvm.objc.clang.arc.noop.use
33 // no longer consumes the function return or is deleted. In that case, it's
34 // not necessary to emit the marker instruction or calls to the ARC runtime
35 // functions.
36 return !CB->getFunctionType()->getReturnType()->isVoidTy() &&
37 CB->getOperandBundle(LLVMContext::OB_clang_arc_attachedcall)
38 .has_value();
39}
40
41/// This function returns operand bundle clang_arc_attachedcall's argument,
42/// which is the address of the ARC runtime function.
43inline std::optional<Function *> getAttachedARCFunction(const CallBase *CB) {
44 auto B = CB->getOperandBundle(LLVMContext::OB_clang_arc_attachedcall);
23
Called C++ object pointer is null
45 if (!B)
46 return std::nullopt;
47
48 return cast<Function>(B->Inputs[0]);
49}
50
51/// Check whether the function is retainRV/unsafeClaimRV.
52inline bool isRetainOrClaimRV(ARCInstKind Kind) {
53 return Kind == ARCInstKind::RetainRV || Kind == ARCInstKind::UnsafeClaimRV;
54}
55
56/// This function returns the ARCInstKind of the function attached to operand
57/// bundle clang_arc_attachedcall. It returns std::nullopt if the call doesn't
58/// have the operand bundle or the operand is null. Otherwise it returns either
59/// RetainRV or UnsafeClaimRV.
60inline ARCInstKind getAttachedARCFunctionKind(const CallBase *CB) {
61 std::optional<Function *> Fn = getAttachedARCFunction(CB);
62 if (!Fn)
63 return ARCInstKind::None;
64 auto FnClass = GetFunctionClass(*Fn);
65 assert(isRetainOrClaimRV(FnClass) && "unexpected ARC runtime function")(static_cast <bool> (isRetainOrClaimRV(FnClass) &&
"unexpected ARC runtime function") ? void (0) : __assert_fail
("isRetainOrClaimRV(FnClass) && \"unexpected ARC runtime function\""
, "llvm/include/llvm/Analysis/ObjCARCUtil.h", 65, __extension__
__PRETTY_FUNCTION__))
;
66 return FnClass;
67}
68
69} // end namespace objcarc
70} // end namespace llvm
71
72#endif