LLVM 17.0.0git
AArch64CallLowering.cpp
Go to the documentation of this file.
1//===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AArch64CallLowering.h"
16#include "AArch64ISelLowering.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "llvm/ADT/ArrayRef.h"
39#include "llvm/IR/Argument.h"
40#include "llvm/IR/Attributes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Type.h"
43#include "llvm/IR/Value.h"
45#include <algorithm>
46#include <cassert>
47#include <cstdint>
48#include <iterator>
49
50#define DEBUG_TYPE "aarch64-call-lowering"
51
52using namespace llvm;
53
55 : CallLowering(&TLI) {}
56
57static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT,
58 MVT &LocVT) {
59 // If ValVT is i1/i8/i16, we should set LocVT to i8/i8/i16. This is a legacy
60 // hack because the DAG calls the assignment function with pre-legalized
61 // register typed values, not the raw type.
62 //
63 // This hack is not applied to return values which are not passed on the
64 // stack.
65 if (OrigVT == MVT::i1 || OrigVT == MVT::i8)
66 ValVT = LocVT = MVT::i8;
67 else if (OrigVT == MVT::i16)
68 ValVT = LocVT = MVT::i16;
69}
70
71// Account for i1/i8/i16 stack passed value hack
73 const MVT ValVT = VA.getValVT();
74 return (ValVT == MVT::i8 || ValVT == MVT::i16) ? LLT(ValVT)
75 : LLT(VA.getLocVT());
76}
77
78namespace {
79
80struct AArch64IncomingValueAssigner
82 AArch64IncomingValueAssigner(CCAssignFn *AssignFn_,
83 CCAssignFn *AssignFnVarArg_)
84 : IncomingValueAssigner(AssignFn_, AssignFnVarArg_) {}
85
86 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
89 CCState &State) override {
90 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
91 return IncomingValueAssigner::assignArg(ValNo, OrigVT, ValVT, LocVT,
92 LocInfo, Info, Flags, State);
93 }
94};
95
96struct AArch64OutgoingValueAssigner
98 const AArch64Subtarget &Subtarget;
99
100 /// Track if this is used for a return instead of function argument
101 /// passing. We apply a hack to i1/i8/i16 stack passed values, but do not use
102 /// stack passed returns for them and cannot apply the type adjustment.
103 bool IsReturn;
104
105 AArch64OutgoingValueAssigner(CCAssignFn *AssignFn_,
106 CCAssignFn *AssignFnVarArg_,
107 const AArch64Subtarget &Subtarget_,
108 bool IsReturn)
109 : OutgoingValueAssigner(AssignFn_, AssignFnVarArg_),
110 Subtarget(Subtarget_), IsReturn(IsReturn) {}
111
112 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
113 CCValAssign::LocInfo LocInfo,
115 CCState &State) override {
116 bool IsCalleeWin = Subtarget.isCallingConvWin64(State.getCallingConv());
117 bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
118
119 bool Res;
120 if (Info.IsFixed && !UseVarArgsCCForFixed) {
121 if (!IsReturn)
122 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
123 Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
124 } else
125 Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, State);
126
128 return Res;
129 }
130};
131
132struct IncomingArgHandler : public CallLowering::IncomingValueHandler {
133 IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
134 : IncomingValueHandler(MIRBuilder, MRI) {}
135
136 Register getStackAddress(uint64_t Size, int64_t Offset,
138 ISD::ArgFlagsTy Flags) override {
139 auto &MFI = MIRBuilder.getMF().getFrameInfo();
140
141 // Byval is assumed to be writable memory, but other stack passed arguments
142 // are not.
143 const bool IsImmutable = !Flags.isByVal();
144
145 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);
146 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
147 auto AddrReg = MIRBuilder.buildFrameIndex(LLT::pointer(0, 64), FI);
148 return AddrReg.getReg(0);
149 }
150
151 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
152 ISD::ArgFlagsTy Flags) const override {
153 // For pointers, we just need to fixup the integer types reported in the
154 // CCValAssign.
155 if (Flags.isPointer())
158 }
159
160 void assignValueToReg(Register ValVReg, Register PhysReg,
161 CCValAssign VA) override {
162 markPhysRegUsed(PhysReg);
163 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
164 }
165
166 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
167 MachinePointerInfo &MPO, CCValAssign &VA) override {
168 MachineFunction &MF = MIRBuilder.getMF();
169
170 LLT ValTy(VA.getValVT());
171 LLT LocTy(VA.getLocVT());
172
173 // Fixup the types for the DAG compatibility hack.
174 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16)
175 std::swap(ValTy, LocTy);
176 else {
177 // The calling code knows if this is a pointer or not, we're only touching
178 // the LocTy for the i8/i16 hack.
179 assert(LocTy.getSizeInBits() == MemTy.getSizeInBits());
180 LocTy = MemTy;
181 }
182
183 auto MMO = MF.getMachineMemOperand(
185 inferAlignFromPtrInfo(MF, MPO));
186
187 switch (VA.getLocInfo()) {
188 case CCValAssign::LocInfo::ZExt:
189 MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, ValVReg, Addr, *MMO);
190 return;
191 case CCValAssign::LocInfo::SExt:
192 MIRBuilder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, ValVReg, Addr, *MMO);
193 return;
194 default:
195 MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
196 return;
197 }
198 }
199
200 /// How the physical register gets marked varies between formal
201 /// parameters (it's a basic-block live-in), and a call instruction
202 /// (it's an implicit-def of the BL).
203 virtual void markPhysRegUsed(MCRegister PhysReg) = 0;
204};
205
206struct FormalArgHandler : public IncomingArgHandler {
208 : IncomingArgHandler(MIRBuilder, MRI) {}
209
210 void markPhysRegUsed(MCRegister PhysReg) override {
211 MIRBuilder.getMRI()->addLiveIn(PhysReg);
212 MIRBuilder.getMBB().addLiveIn(PhysReg);
213 }
214};
215
216struct CallReturnHandler : public IncomingArgHandler {
219 : IncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
220
221 void markPhysRegUsed(MCRegister PhysReg) override {
222 MIB.addDef(PhysReg, RegState::Implicit);
223 }
224
226};
227
228/// A special return arg handler for "returned" attribute arg calls.
229struct ReturnedArgCallReturnHandler : public CallReturnHandler {
230 ReturnedArgCallReturnHandler(MachineIRBuilder &MIRBuilder,
233 : CallReturnHandler(MIRBuilder, MRI, MIB) {}
234
235 void markPhysRegUsed(MCRegister PhysReg) override {}
236};
237
238struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
239 OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
240 MachineInstrBuilder MIB, bool IsTailCall = false,
241 int FPDiff = 0)
242 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), IsTailCall(IsTailCall),
243 FPDiff(FPDiff),
244 Subtarget(MIRBuilder.getMF().getSubtarget<AArch64Subtarget>()) {}
245
246 Register getStackAddress(uint64_t Size, int64_t Offset,
248 ISD::ArgFlagsTy Flags) override {
249 MachineFunction &MF = MIRBuilder.getMF();
250 LLT p0 = LLT::pointer(0, 64);
251 LLT s64 = LLT::scalar(64);
252
253 if (IsTailCall) {
254 assert(!Flags.isByVal() && "byval unhandled with tail calls");
255
256 Offset += FPDiff;
257 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
258 auto FIReg = MIRBuilder.buildFrameIndex(p0, FI);
260 return FIReg.getReg(0);
261 }
262
263 if (!SPReg)
264 SPReg = MIRBuilder.buildCopy(p0, Register(AArch64::SP)).getReg(0);
265
266 auto OffsetReg = MIRBuilder.buildConstant(s64, Offset);
267
268 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg);
269
271 return AddrReg.getReg(0);
272 }
273
274 /// We need to fixup the reported store size for certain value types because
275 /// we invert the interpretation of ValVT and LocVT in certain cases. This is
276 /// for compatability with the DAG call lowering implementation, which we're
277 /// currently building on top of.
278 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
279 ISD::ArgFlagsTy Flags) const override {
280 if (Flags.isPointer())
283 }
284
285 void assignValueToReg(Register ValVReg, Register PhysReg,
286 CCValAssign VA) override {
287 MIB.addUse(PhysReg, RegState::Implicit);
288 Register ExtReg = extendRegister(ValVReg, VA);
289 MIRBuilder.buildCopy(PhysReg, ExtReg);
290 }
291
292 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
293 MachinePointerInfo &MPO, CCValAssign &VA) override {
294 MachineFunction &MF = MIRBuilder.getMF();
295 auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,
296 inferAlignFromPtrInfo(MF, MPO));
297 MIRBuilder.buildStore(ValVReg, Addr, *MMO);
298 }
299
300 void assignValueToAddress(const CallLowering::ArgInfo &Arg, unsigned RegIndex,
301 Register Addr, LLT MemTy, MachinePointerInfo &MPO,
302 CCValAssign &VA) override {
303 unsigned MaxSize = MemTy.getSizeInBytes() * 8;
304 // For varargs, we always want to extend them to 8 bytes, in which case
305 // we disable setting a max.
306 if (!Arg.IsFixed)
307 MaxSize = 0;
308
309 Register ValVReg = Arg.Regs[RegIndex];
310 if (VA.getLocInfo() != CCValAssign::LocInfo::FPExt) {
311 MVT LocVT = VA.getLocVT();
312 MVT ValVT = VA.getValVT();
313
314 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16) {
315 std::swap(ValVT, LocVT);
316 MemTy = LLT(VA.getValVT());
317 }
318
319 ValVReg = extendRegister(ValVReg, VA, MaxSize);
320 } else {
321 // The store does not cover the full allocated stack slot.
322 MemTy = LLT(VA.getValVT());
323 }
324
325 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
326 }
327
329
330 bool IsTailCall;
331
332 /// For tail calls, the byte offset of the call's argument area from the
333 /// callee's. Unused elsewhere.
334 int FPDiff;
335
336 // Cache the SP register vreg if we need it more than once in this call site.
337 Register SPReg;
338
339 const AArch64Subtarget &Subtarget;
340};
341} // namespace
342
343static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
344 return (CallConv == CallingConv::Fast && TailCallOpt) ||
345 CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;
346}
347
349 const Value *Val,
350 ArrayRef<Register> VRegs,
352 Register SwiftErrorVReg) const {
353 auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR);
354 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&
355 "Return value without a vreg");
356
357 bool Success = true;
358 if (!FLI.CanLowerReturn) {
359 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister);
360 } else if (!VRegs.empty()) {
361 MachineFunction &MF = MIRBuilder.getMF();
362 const Function &F = MF.getFunction();
363 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
364
366 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
367 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
368 auto &DL = F.getParent()->getDataLayout();
369 LLVMContext &Ctx = Val->getType()->getContext();
370
371 SmallVector<EVT, 4> SplitEVTs;
372 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
373 assert(VRegs.size() == SplitEVTs.size() &&
374 "For each split Type there should be exactly one VReg.");
375
376 SmallVector<ArgInfo, 8> SplitArgs;
377 CallingConv::ID CC = F.getCallingConv();
378
379 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
380 Register CurVReg = VRegs[i];
381 ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx), 0};
383
384 // i1 is a special case because SDAG i1 true is naturally zero extended
385 // when widened using ANYEXT. We need to do it explicitly here.
386 auto &Flags = CurArgInfo.Flags[0];
387 if (MRI.getType(CurVReg).getSizeInBits() == 1 && !Flags.isSExt() &&
388 !Flags.isZExt()) {
389 CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0);
390 } else if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) ==
391 1) {
392 // Some types will need extending as specified by the CC.
393 MVT NewVT = TLI.getRegisterTypeForCallingConv(Ctx, CC, SplitEVTs[i]);
394 if (EVT(NewVT) != SplitEVTs[i]) {
395 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
396 if (F.getAttributes().hasRetAttr(Attribute::SExt))
397 ExtendOp = TargetOpcode::G_SEXT;
398 else if (F.getAttributes().hasRetAttr(Attribute::ZExt))
399 ExtendOp = TargetOpcode::G_ZEXT;
400
401 LLT NewLLT(NewVT);
402 LLT OldLLT(MVT::getVT(CurArgInfo.Ty));
403 CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Ctx);
404 // Instead of an extend, we might have a vector type which needs
405 // padding with more elements, e.g. <2 x half> -> <4 x half>.
406 if (NewVT.isVector()) {
407 if (OldLLT.isVector()) {
408 if (NewLLT.getNumElements() > OldLLT.getNumElements()) {
409 // We don't handle VA types which are not exactly twice the
410 // size, but can easily be done in future.
411 if (NewLLT.getNumElements() != OldLLT.getNumElements() * 2) {
412 LLVM_DEBUG(dbgs() << "Outgoing vector ret has too many elts");
413 return false;
414 }
415 auto Undef = MIRBuilder.buildUndef({OldLLT});
416 CurVReg =
417 MIRBuilder.buildMergeLikeInstr({NewLLT}, {CurVReg, Undef})
418 .getReg(0);
419 } else {
420 // Just do a vector extend.
421 CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
422 .getReg(0);
423 }
424 } else if (NewLLT.getNumElements() == 2) {
425 // We need to pad a <1 x S> type to <2 x S>. Since we don't have
426 // <1 x S> vector types in GISel we use a build_vector instead
427 // of a vector merge/concat.
428 auto Undef = MIRBuilder.buildUndef({OldLLT});
429 CurVReg =
430 MIRBuilder
431 .buildBuildVector({NewLLT}, {CurVReg, Undef.getReg(0)})
432 .getReg(0);
433 } else {
434 LLVM_DEBUG(dbgs() << "Could not handle ret ty\n");
435 return false;
436 }
437 } else {
438 // If the split EVT was a <1 x T> vector, and NewVT is T, then we
439 // don't have to do anything since we don't distinguish between the
440 // two.
441 if (NewLLT != MRI.getType(CurVReg)) {
442 // A scalar extend.
443 CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
444 .getReg(0);
445 }
446 }
447 }
448 }
449 if (CurVReg != CurArgInfo.Regs[0]) {
450 CurArgInfo.Regs[0] = CurVReg;
451 // Reset the arg flags after modifying CurVReg.
453 }
454 splitToValueTypes(CurArgInfo, SplitArgs, DL, CC);
455 }
456
457 AArch64OutgoingValueAssigner Assigner(AssignFn, AssignFn, Subtarget,
458 /*IsReturn*/ true);
459 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB);
460 Success = determineAndHandleAssignments(Handler, Assigner, SplitArgs,
461 MIRBuilder, CC, F.isVarArg());
462 }
463
464 if (SwiftErrorVReg) {
465 MIB.addUse(AArch64::X21, RegState::Implicit);
466 MIRBuilder.buildCopy(AArch64::X21, SwiftErrorVReg);
467 }
468
469 MIRBuilder.insertInstr(MIB);
470 return Success;
471}
472
474 CallingConv::ID CallConv,
476 bool IsVarArg) const {
478 const auto &TLI = *getTLI<AArch64TargetLowering>();
479 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
480 MF.getFunction().getContext());
481
482 return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv));
483}
484
485/// Helper function to compute forwarded registers for musttail calls. Computes
486/// the forwarded registers, sets MBB liveness, and emits COPY instructions that
487/// can be used to save + restore registers later.
489 CCAssignFn *AssignFn) {
490 MachineBasicBlock &MBB = MIRBuilder.getMBB();
491 MachineFunction &MF = MIRBuilder.getMF();
492 MachineFrameInfo &MFI = MF.getFrameInfo();
493
494 if (!MFI.hasMustTailInVarArgFunc())
495 return;
496
498 const Function &F = MF.getFunction();
499 assert(F.isVarArg() && "Expected F to be vararg?");
500
501 // Compute the set of forwarded registers. The rest are scratch.
503 CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
504 F.getContext());
505 SmallVector<MVT, 2> RegParmTypes;
506 RegParmTypes.push_back(MVT::i64);
507 RegParmTypes.push_back(MVT::f128);
508
509 // Later on, we can use this vector to restore the registers if necessary.
511 FuncInfo->getForwardedMustTailRegParms();
512 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, AssignFn);
513
514 // Conservatively forward X8, since it might be used for an aggregate
515 // return.
516 if (!CCInfo.isAllocated(AArch64::X8)) {
517 Register X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
518 Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
519 }
520
521 // Add the forwards to the MachineBasicBlock and MachineFunction.
522 for (const auto &F : Forwards) {
523 MBB.addLiveIn(F.PReg);
524 MIRBuilder.buildCopy(Register(F.VReg), Register(F.PReg));
525 }
526}
527
529 auto &F = MF.getFunction();
530 if (F.getReturnType()->isScalableTy() ||
531 llvm::any_of(F.args(), [](const Argument &A) {
532 return A.getType()->isScalableTy();
533 }))
534 return true;
535 const auto &ST = MF.getSubtarget<AArch64Subtarget>();
536 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
537 LLVM_DEBUG(dbgs() << "Falling back to SDAG because we don't support no-NEON\n");
538 return true;
539 }
540
541 SMEAttrs Attrs(F);
542 if (Attrs.hasNewZAInterface() ||
543 (!Attrs.hasStreamingInterface() && Attrs.hasStreamingBody()))
544 return true;
545
546 return false;
547}
548
549void AArch64CallLowering::saveVarArgRegisters(
551 CCState &CCInfo) const {
554
555 MachineFunction &MF = MIRBuilder.getMF();
557 MachineFrameInfo &MFI = MF.getFrameInfo();
559 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
560 bool IsWin64CC =
561 Subtarget.isCallingConvWin64(CCInfo.getCallingConv());
562 const LLT p0 = LLT::pointer(0, 64);
563 const LLT s64 = LLT::scalar(64);
564
565 unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
566 unsigned NumVariadicGPRArgRegs = GPRArgRegs.size() - FirstVariadicGPR + 1;
567
568 unsigned GPRSaveSize = 8 * (GPRArgRegs.size() - FirstVariadicGPR);
569 int GPRIdx = 0;
570 if (GPRSaveSize != 0) {
571 if (IsWin64CC) {
572 GPRIdx = MFI.CreateFixedObject(GPRSaveSize,
573 -static_cast<int>(GPRSaveSize), false);
574 } else
575 GPRIdx = MFI.CreateStackObject(GPRSaveSize, Align(8), false);
576
577 auto FIN = MIRBuilder.buildFrameIndex(p0, GPRIdx);
578 auto Offset =
579 MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 8);
580
581 for (unsigned i = FirstVariadicGPR; i < GPRArgRegs.size(); ++i) {
582 Register Val = MRI.createGenericVirtualRegister(s64);
583 Handler.assignValueToReg(
584 Val, GPRArgRegs[i],
587 auto MPO = IsWin64CC ? MachinePointerInfo::getFixedStack(
588 MF, GPRIdx, (i - FirstVariadicGPR) * 8)
589 : MachinePointerInfo::getStack(MF, i * 8);
590 MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
591
592 FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
593 FIN.getReg(0), Offset);
594 }
595 }
596 FuncInfo->setVarArgsGPRIndex(GPRIdx);
597 FuncInfo->setVarArgsGPRSize(GPRSaveSize);
598
599 if (Subtarget.hasFPARMv8() && !IsWin64CC) {
600 unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
601
602 unsigned FPRSaveSize = 16 * (FPRArgRegs.size() - FirstVariadicFPR);
603 int FPRIdx = 0;
604 if (FPRSaveSize != 0) {
605 FPRIdx = MFI.CreateStackObject(FPRSaveSize, Align(16), false);
606
607 auto FIN = MIRBuilder.buildFrameIndex(p0, FPRIdx);
608 auto Offset =
609 MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 16);
610
611 for (unsigned i = FirstVariadicFPR; i < FPRArgRegs.size(); ++i) {
612 Register Val = MRI.createGenericVirtualRegister(LLT::scalar(128));
613 Handler.assignValueToReg(
614 Val, FPRArgRegs[i],
616 i + MF.getFunction().getNumOperands() + NumVariadicGPRArgRegs,
618
619 auto MPO = MachinePointerInfo::getStack(MF, i * 16);
620 MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
621
622 FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
623 FIN.getReg(0), Offset);
624 }
625 }
626 FuncInfo->setVarArgsFPRIndex(FPRIdx);
627 FuncInfo->setVarArgsFPRSize(FPRSaveSize);
628 }
629}
630
632 MachineIRBuilder &MIRBuilder, const Function &F,
634 MachineFunction &MF = MIRBuilder.getMF();
635 MachineBasicBlock &MBB = MIRBuilder.getMBB();
637 auto &DL = F.getParent()->getDataLayout();
638 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
639 // TODO: Support Arm64EC
640 bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv()) && !Subtarget.isWindowsArm64EC();
641
642 SmallVector<ArgInfo, 8> SplitArgs;
644
645 // Insert the hidden sret parameter if the return value won't fit in the
646 // return registers.
647 if (!FLI.CanLowerReturn)
649
650 unsigned i = 0;
651 for (auto &Arg : F.args()) {
652 if (DL.getTypeStoreSize(Arg.getType()).isZero())
653 continue;
654
655 ArgInfo OrigArg{VRegs[i], Arg, i};
657
658 // i1 arguments are zero-extended to i8 by the caller. Emit a
659 // hint to reflect this.
660 if (OrigArg.Ty->isIntegerTy(1)) {
661 assert(OrigArg.Regs.size() == 1 &&
662 MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
663 "Unexpected registers used for i1 arg");
664
665 auto &Flags = OrigArg.Flags[0];
666 if (!Flags.isZExt() && !Flags.isSExt()) {
667 // Lower i1 argument as i8, and insert AssertZExt + Trunc later.
668 Register OrigReg = OrigArg.Regs[0];
669 Register WideReg = MRI.createGenericVirtualRegister(LLT::scalar(8));
670 OrigArg.Regs[0] = WideReg;
671 BoolArgs.push_back({OrigReg, WideReg});
672 }
673 }
674
675 if (Arg.hasAttribute(Attribute::SwiftAsync))
676 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
677
678 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());
679 ++i;
680 }
681
682 if (!MBB.empty())
683 MIRBuilder.setInstr(*MBB.begin());
684
685 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
686 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(F.getCallingConv(), IsWin64 && F.isVarArg());
687
688 AArch64IncomingValueAssigner Assigner(AssignFn, AssignFn);
689 FormalArgHandler Handler(MIRBuilder, MRI);
691 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
692 if (!determineAssignments(Assigner, SplitArgs, CCInfo) ||
693 !handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, MIRBuilder))
694 return false;
695
696 if (!BoolArgs.empty()) {
697 for (auto &KV : BoolArgs) {
698 Register OrigReg = KV.first;
699 Register WideReg = KV.second;
700 LLT WideTy = MRI.getType(WideReg);
701 assert(MRI.getType(OrigReg).getScalarSizeInBits() == 1 &&
702 "Unexpected bit size of a bool arg");
703 MIRBuilder.buildTrunc(
704 OrigReg, MIRBuilder.buildAssertZExt(WideTy, WideReg, 1).getReg(0));
705 }
706 }
707
709 uint64_t StackOffset = Assigner.StackOffset;
710 if (F.isVarArg()) {
711 if ((!Subtarget.isTargetDarwin() && !Subtarget.isWindowsArm64EC()) || IsWin64) {
712 // The AAPCS variadic function ABI is identical to the non-variadic
713 // one. As a result there may be more arguments in registers and we should
714 // save them for future reference.
715 // Win64 variadic functions also pass arguments in registers, but all
716 // float arguments are passed in integer registers.
717 saveVarArgRegisters(MIRBuilder, Handler, CCInfo);
718 } else if (Subtarget.isWindowsArm64EC()) {
719 return false;
720 }
721
722 // We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
724 alignTo(Assigner.StackOffset, Subtarget.isTargetILP32() ? 4 : 8);
725
726 auto &MFI = MIRBuilder.getMF().getFrameInfo();
727 FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
728 }
729
730 if (doesCalleeRestoreStack(F.getCallingConv(),
732 // We have a non-standard ABI, so why not make full use of the stack that
733 // we're going to pop? It must be aligned to 16 B in any case.
735
736 // If we're expected to restore the stack (e.g. fastcc), then we'll be
737 // adding a multiple of 16.
738 FuncInfo->setArgumentStackToRestore(StackOffset);
739
740 // Our own callers will guarantee that the space is free by giving an
741 // aligned value to CALLSEQ_START.
742 }
743
744 // When we tail call, we need to check if the callee's arguments
745 // will fit on the caller's stack. So, whenever we lower formal arguments,
746 // we should keep track of this information, since we might lower a tail call
747 // in this function later.
748 FuncInfo->setBytesInStackArgArea(StackOffset);
749
750 if (Subtarget.hasCustomCallingConv())
751 Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
752
753 handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
754
755 // Move back to the end of the basic block.
756 MIRBuilder.setMBB(MBB);
757
758 return true;
759}
760
761/// Return true if the calling convention is one that we can guarantee TCO for.
762static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
763 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
765}
766
767/// Return true if we might ever do TCO for calls with this calling convention.
769 switch (CC) {
770 case CallingConv::C:
776 return true;
777 default:
778 return false;
779 }
780}
781
782/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
783/// CC.
784static std::pair<CCAssignFn *, CCAssignFn *>
786 return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
787}
788
789bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
790 CallLoweringInfo &Info, MachineFunction &MF,
791 SmallVectorImpl<ArgInfo> &InArgs) const {
792 const Function &CallerF = MF.getFunction();
793 CallingConv::ID CalleeCC = Info.CallConv;
794 CallingConv::ID CallerCC = CallerF.getCallingConv();
795
796 // If the calling conventions match, then everything must be the same.
797 if (CalleeCC == CallerCC)
798 return true;
799
800 // Check if the caller and callee will handle arguments in the same way.
801 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
802 CCAssignFn *CalleeAssignFnFixed;
803 CCAssignFn *CalleeAssignFnVarArg;
804 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
805 getAssignFnsForCC(CalleeCC, TLI);
806
807 CCAssignFn *CallerAssignFnFixed;
808 CCAssignFn *CallerAssignFnVarArg;
809 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
810 getAssignFnsForCC(CallerCC, TLI);
811
812 AArch64IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
813 CalleeAssignFnVarArg);
814 AArch64IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
815 CallerAssignFnVarArg);
816
817 if (!resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner))
818 return false;
819
820 // Make sure that the caller and callee preserve all of the same registers.
821 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
822 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
823 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
825 TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
826 TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
827 }
828
829 return TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved);
830}
831
832bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
833 CallLoweringInfo &Info, MachineFunction &MF,
834 SmallVectorImpl<ArgInfo> &OutArgs) const {
835 // If there are no outgoing arguments, then we are done.
836 if (OutArgs.empty())
837 return true;
838
839 const Function &CallerF = MF.getFunction();
840 LLVMContext &Ctx = CallerF.getContext();
841 CallingConv::ID CalleeCC = Info.CallConv;
842 CallingConv::ID CallerCC = CallerF.getCallingConv();
843 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
844 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
845
846 CCAssignFn *AssignFnFixed;
847 CCAssignFn *AssignFnVarArg;
848 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
849
850 // We have outgoing arguments. Make sure that we can tail call with them.
852 CCState OutInfo(CalleeCC, false, MF, OutLocs, Ctx);
853
854 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
855 Subtarget, /*IsReturn*/ false);
856 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo)) {
857 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
858 return false;
859 }
860
861 // Make sure that they can fit on the caller's stack.
863 if (OutInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea()) {
864 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
865 return false;
866 }
867
868 // Verify that the parameters in callee-saved registers match.
869 // TODO: Port this over to CallLowering as general code once swiftself is
870 // supported.
871 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
872 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
874
875 if (Info.IsVarArg) {
876 // Be conservative and disallow variadic memory operands to match SDAG's
877 // behaviour.
878 // FIXME: If the caller's calling convention is C, then we can
879 // potentially use its argument area. However, for cases like fastcc,
880 // we can't do anything.
881 for (unsigned i = 0; i < OutLocs.size(); ++i) {
882 auto &ArgLoc = OutLocs[i];
883 if (ArgLoc.isRegLoc())
884 continue;
885
887 dbgs()
888 << "... Cannot tail call vararg function with stack arguments\n");
889 return false;
890 }
891 }
892
893 return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);
894}
895
897 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
899 SmallVectorImpl<ArgInfo> &OutArgs) const {
900
901 // Must pass all target-independent checks in order to tail call optimize.
902 if (!Info.IsTailCall)
903 return false;
904
905 CallingConv::ID CalleeCC = Info.CallConv;
906 MachineFunction &MF = MIRBuilder.getMF();
907 const Function &CallerF = MF.getFunction();
908
909 LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
910
911 if (Info.SwiftErrorVReg) {
912 // TODO: We should handle this.
913 // Note that this is also handled by the check for no outgoing arguments.
914 // Proactively disabling this though, because the swifterror handling in
915 // lowerCall inserts a COPY *after* the location of the call.
916 LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n");
917 return false;
918 }
919
920 if (!mayTailCallThisCC(CalleeCC)) {
921 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
922 return false;
923 }
924
925 // Byval parameters hand the function a pointer directly into the stack area
926 // we want to reuse during a tail call. Working around this *is* possible (see
927 // X86).
928 //
929 // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
930 // it?
931 //
932 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
933 // In this case, it is necessary to save/restore X0 in the callee. Tail
934 // call opt interferes with this. So we disable tail call opt when the
935 // caller has an argument with "inreg" attribute.
936 //
937 // FIXME: Check whether the callee also has an "inreg" argument.
938 //
939 // When the caller has a swifterror argument, we don't want to tail call
940 // because would have to move into the swifterror register before the
941 // tail call.
942 if (any_of(CallerF.args(), [](const Argument &A) {
943 return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
944 })) {
945 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
946 "inreg, or swifterror arguments\n");
947 return false;
948 }
949
950 // Externally-defined functions with weak linkage should not be
951 // tail-called on AArch64 when the OS does not support dynamic
952 // pre-emption of symbols, as the AAELF spec requires normal calls
953 // to undefined weak functions to be replaced with a NOP or jump to the
954 // next instruction. The behaviour of branch instructions in this
955 // situation (as used for tail calls) is implementation-defined, so we
956 // cannot rely on the linker replacing the tail call with a return.
957 if (Info.Callee.isGlobal()) {
958 const GlobalValue *GV = Info.Callee.getGlobal();
959 const Triple &TT = MF.getTarget().getTargetTriple();
960 if (GV->hasExternalWeakLinkage() &&
961 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
962 TT.isOSBinFormatMachO())) {
963 LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "
964 "with weak linkage for this OS.\n");
965 return false;
966 }
967 }
968
969 // If we have -tailcallopt, then we're done.
971 return CalleeCC == CallerF.getCallingConv();
972
973 // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
974 // Try to find cases where we can do that.
975
976 // I want anyone implementing a new calling convention to think long and hard
977 // about this assert.
978 assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
979 "Unexpected variadic calling convention");
980
981 // Verify that the incoming and outgoing arguments from the callee are
982 // safe to tail call.
983 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
985 dbgs()
986 << "... Caller and callee have incompatible calling conventions.\n");
987 return false;
988 }
989
990 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
991 return false;
992
994 dbgs() << "... Call is eligible for tail call optimization.\n");
995 return true;
996}
997
998static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
999 bool IsTailCall) {
1000 if (!IsTailCall)
1001 return IsIndirect ? getBLRCallOpcode(CallerF) : (unsigned)AArch64::BL;
1002
1003 if (!IsIndirect)
1004 return AArch64::TCRETURNdi;
1005
1006 // When BTI is enabled, we need to use TCRETURNriBTI to make sure that we use
1007 // x16 or x17.
1009 return AArch64::TCRETURNriBTI;
1010
1011 return AArch64::TCRETURNri;
1012}
1013
1014static const uint32_t *
1018 const uint32_t *Mask;
1019 if (!OutArgs.empty() && OutArgs[0].Flags[0].isReturned()) {
1020 // For 'this' returns, use the X0-preserving mask if applicable
1021 Mask = TRI.getThisReturnPreservedMask(MF, Info.CallConv);
1022 if (!Mask) {
1023 OutArgs[0].Flags[0].setReturned(false);
1024 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1025 }
1026 } else {
1027 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1028 }
1029 return Mask;
1030}
1031
1032bool AArch64CallLowering::lowerTailCall(
1033 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1034 SmallVectorImpl<ArgInfo> &OutArgs) const {
1035 MachineFunction &MF = MIRBuilder.getMF();
1036 const Function &F = MF.getFunction();
1038 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1040
1041 // True when we're tail calling, but without -tailcallopt.
1042 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt &&
1043 Info.CallConv != CallingConv::Tail &&
1044 Info.CallConv != CallingConv::SwiftTail;
1045
1046 // TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
1047 // register class. Until we can do that, we should fall back here.
1049 LLVM_DEBUG(
1050 dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n");
1051 return false;
1052 }
1053
1054 // Find out which ABI gets to decide where things go.
1055 CallingConv::ID CalleeCC = Info.CallConv;
1056 CCAssignFn *AssignFnFixed;
1057 CCAssignFn *AssignFnVarArg;
1058 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
1059
1060 MachineInstrBuilder CallSeqStart;
1061 if (!IsSibCall)
1062 CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1063
1064 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true);
1065 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1066 MIB.add(Info.Callee);
1067
1068 // Byte offset for the tail call. When we are sibcalling, this will always
1069 // be 0.
1070 MIB.addImm(0);
1071
1072 // Tell the call which registers are clobbered.
1073 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1074 auto TRI = Subtarget.getRegisterInfo();
1075 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1076 if (Subtarget.hasCustomCallingConv())
1077 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1078 MIB.addRegMask(Mask);
1079
1080 if (Info.CFIType)
1081 MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1082
1083 if (TRI->isAnyArgRegReserved(MF))
1084 TRI->emitReservedArgRegCallError(MF);
1085
1086 // FPDiff is the byte offset of the call's argument area from the callee's.
1087 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1088 // by this amount for a tail call. In a sibling call it must be 0 because the
1089 // caller will deallocate the entire stack and the callee still expects its
1090 // arguments to begin at SP+0.
1091 int FPDiff = 0;
1092
1093 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1094 // by -tailcallopt. For sibcalls, the memory operands for the call are
1095 // already available in the caller's incoming argument space.
1096 unsigned NumBytes = 0;
1097 if (!IsSibCall) {
1098 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1099 // before handling assignments, because FPDiff must be known for memory
1100 // arguments.
1101 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1103 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1104
1105 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
1106 Subtarget, /*IsReturn*/ false);
1107 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo))
1108 return false;
1109
1110 // The callee will pop the argument stack as a tail call. Thus, we must
1111 // keep it 16-byte aligned.
1112 NumBytes = alignTo(OutInfo.getNextStackOffset(), 16);
1113
1114 // FPDiff will be negative if this tail call requires more space than we
1115 // would automatically have in our incoming argument space. Positive if we
1116 // actually shrink the stack.
1117 FPDiff = NumReusableBytes - NumBytes;
1118
1119 // Update the required reserved area if this is the tail call requiring the
1120 // most argument stack space.
1121 if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff)
1122 FuncInfo->setTailCallReservedStack(-FPDiff);
1123
1124 // The stack pointer must be 16-byte aligned at all times it's used for a
1125 // memory operation, which in practice means at *all* times and in
1126 // particular across call boundaries. Therefore our own arguments started at
1127 // a 16-byte aligned SP and the delta applied for the tail call should
1128 // satisfy the same constraint.
1129 assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
1130 }
1131
1132 const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
1133
1134 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1135 Subtarget, /*IsReturn*/ false);
1136
1137 // Do the actual argument marshalling.
1138 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB,
1139 /*IsTailCall*/ true, FPDiff);
1140 if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1141 CalleeCC, Info.IsVarArg))
1142 return false;
1143
1144 Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1145
1146 if (Info.IsVarArg && Info.IsMustTailCall) {
1147 // Now we know what's being passed to the function. Add uses to the call for
1148 // the forwarded registers that we *aren't* passing as parameters. This will
1149 // preserve the copies we build earlier.
1150 for (const auto &F : Forwards) {
1151 Register ForwardedReg = F.PReg;
1152 // If the register is already passed, or aliases a register which is
1153 // already being passed, then skip it.
1154 if (any_of(MIB->uses(), [&ForwardedReg, &TRI](const MachineOperand &Use) {
1155 if (!Use.isReg())
1156 return false;
1157 return TRI->regsOverlap(Use.getReg(), ForwardedReg);
1158 }))
1159 continue;
1160
1161 // We aren't passing it already, so we should add it to the call.
1162 MIRBuilder.buildCopy(ForwardedReg, Register(F.VReg));
1163 MIB.addReg(ForwardedReg, RegState::Implicit);
1164 }
1165 }
1166
1167 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1168 // sequence start and end here.
1169 if (!IsSibCall) {
1170 MIB->getOperand(1).setImm(FPDiff);
1171 CallSeqStart.addImm(0).addImm(0);
1172 // End the call sequence *before* emitting the call. Normally, we would
1173 // tidy the frame up after the call. However, here, we've laid out the
1174 // parameters so that when SP is reset, they will be in the correct
1175 // location.
1176 MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP).addImm(0).addImm(0);
1177 }
1178
1179 // Now we can add the actual call instruction to the correct basic block.
1180 MIRBuilder.insertInstr(MIB);
1181
1182 // If Callee is a reg, since it is used by a target specific instruction,
1183 // it must have a register class matching the constraint of that instruction.
1184 if (MIB->getOperand(0).isReg())
1186 *MF.getSubtarget().getRegBankInfo(), *MIB,
1187 MIB->getDesc(), MIB->getOperand(0), 0);
1188
1190 Info.LoweredTailCall = true;
1191 return true;
1192}
1193
1195 CallLoweringInfo &Info) const {
1196 MachineFunction &MF = MIRBuilder.getMF();
1197 const Function &F = MF.getFunction();
1199 auto &DL = F.getParent()->getDataLayout();
1200 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1201 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1202
1203 // Arm64EC has extra requirements for varargs calls; bail out for now.
1204 if (Info.IsVarArg && Subtarget.isWindowsArm64EC())
1205 return false;
1206
1208 for (auto &OrigArg : Info.OrigArgs) {
1209 splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);
1210 // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
1211 auto &Flags = OrigArg.Flags[0];
1212 if (OrigArg.Ty->isIntegerTy(1) && !Flags.isSExt() && !Flags.isZExt()) {
1213 ArgInfo &OutArg = OutArgs.back();
1214 assert(OutArg.Regs.size() == 1 &&
1215 MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 &&
1216 "Unexpected registers used for i1 arg");
1217
1218 // We cannot use a ZExt ArgInfo flag here, because it will
1219 // zero-extend the argument to i32 instead of just i8.
1220 OutArg.Regs[0] =
1221 MIRBuilder.buildZExt(LLT::scalar(8), OutArg.Regs[0]).getReg(0);
1222 LLVMContext &Ctx = MF.getFunction().getContext();
1223 OutArg.Ty = Type::getInt8Ty(Ctx);
1224 }
1225 }
1226
1228 if (!Info.OrigRet.Ty->isVoidTy())
1229 splitToValueTypes(Info.OrigRet, InArgs, DL, Info.CallConv);
1230
1231 // If we can lower as a tail call, do that instead.
1232 bool CanTailCallOpt =
1233 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
1234
1235 // We must emit a tail call if we have musttail.
1236 if (Info.IsMustTailCall && !CanTailCallOpt) {
1237 // There are types of incoming/outgoing arguments we can't handle yet, so
1238 // it doesn't make sense to actually die here like in ISelLowering. Instead,
1239 // fall back to SelectionDAG and let it try to handle this.
1240 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
1241 return false;
1242 }
1243
1244 Info.IsTailCall = CanTailCallOpt;
1245 if (CanTailCallOpt)
1246 return lowerTailCall(MIRBuilder, Info, OutArgs);
1247
1248 // Find out which ABI gets to decide where things go.
1249 CCAssignFn *AssignFnFixed;
1250 CCAssignFn *AssignFnVarArg;
1251 std::tie(AssignFnFixed, AssignFnVarArg) =
1252 getAssignFnsForCC(Info.CallConv, TLI);
1253
1254 MachineInstrBuilder CallSeqStart;
1255 CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1256
1257 // Create a temporarily-floating call instruction so we can add the implicit
1258 // uses of arg registers.
1259
1260 unsigned Opc = 0;
1261 // Calls with operand bundle "clang.arc.attachedcall" are special. They should
1262 // be expanded to the call, directly followed by a special marker sequence and
1263 // a call to an ObjC library function.
1265 Opc = AArch64::BLR_RVMARKER;
1266 // A call to a returns twice function like setjmp must be followed by a bti
1267 // instruction.
1268 else if (Info.CB && Info.CB->hasFnAttr(Attribute::ReturnsTwice) &&
1269 !Subtarget.noBTIAtReturnTwice() &&
1271 Opc = AArch64::BLR_BTI;
1272 else
1273 Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
1274
1275 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1276 unsigned CalleeOpNo = 0;
1277
1278 if (Opc == AArch64::BLR_RVMARKER) {
1279 // Add a target global address for the retainRV/claimRV runtime function
1280 // just before the call target.
1282 MIB.addGlobalAddress(ARCFn);
1283 ++CalleeOpNo;
1284 } else if (Info.CFIType) {
1285 MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1286 }
1287
1288 MIB.add(Info.Callee);
1289
1290 // Tell the call which registers are clobbered.
1291 const uint32_t *Mask;
1292 const auto *TRI = Subtarget.getRegisterInfo();
1293
1294 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1295 Subtarget, /*IsReturn*/ false);
1296 // Do the actual argument marshalling.
1297 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, /*IsReturn*/ false);
1298 if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1299 Info.CallConv, Info.IsVarArg))
1300 return false;
1301
1302 Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1303
1305 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1306 MIB.addRegMask(Mask);
1307
1308 if (TRI->isAnyArgRegReserved(MF))
1309 TRI->emitReservedArgRegCallError(MF);
1310
1311 // Now we can add the actual call instruction to the correct basic block.
1312 MIRBuilder.insertInstr(MIB);
1313
1314 uint64_t CalleePopBytes =
1317 ? alignTo(Assigner.StackOffset, 16)
1318 : 0;
1319
1320 CallSeqStart.addImm(Assigner.StackOffset).addImm(0);
1321 MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
1322 .addImm(Assigner.StackOffset)
1323 .addImm(CalleePopBytes);
1324
1325 // If Callee is a reg, since it is used by a target specific
1326 // instruction, it must have a register class matching the
1327 // constraint of that instruction.
1328 if (MIB->getOperand(CalleeOpNo).isReg())
1329 constrainOperandRegClass(MF, *TRI, MRI, *Subtarget.getInstrInfo(),
1330 *Subtarget.getRegBankInfo(), *MIB, MIB->getDesc(),
1331 MIB->getOperand(CalleeOpNo), CalleeOpNo);
1332
1333 // Finally we can copy the returned value back into its virtual-register. In
1334 // symmetry with the arguments, the physical register must be an
1335 // implicit-define of the call instruction.
1336 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1337 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv);
1338 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1339 bool UsingReturnedArg =
1340 !OutArgs.empty() && OutArgs[0].Flags[0].isReturned();
1341
1342 AArch64OutgoingValueAssigner Assigner(RetAssignFn, RetAssignFn, Subtarget,
1343 /*IsReturn*/ false);
1344 ReturnedArgCallReturnHandler ReturnedArgHandler(MIRBuilder, MRI, MIB);
1346 UsingReturnedArg ? ReturnedArgHandler : Handler, Assigner, InArgs,
1347 MIRBuilder, Info.CallConv, Info.IsVarArg,
1348 UsingReturnedArg ? ArrayRef(OutArgs[0].Regs) : std::nullopt))
1349 return false;
1350 }
1351
1352 if (Info.SwiftErrorVReg) {
1353 MIB.addDef(AArch64::X21, RegState::Implicit);
1354 MIRBuilder.buildCopy(Info.SwiftErrorVReg, Register(AArch64::X21));
1355 }
1356
1357 if (!Info.CanLowerReturn) {
1358 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,
1359 Info.DemoteRegister, Info.DemoteStackIndex);
1360 }
1361 return true;
1362}
1363
1365 return Ty.getSizeInBits() == 64;
1366}
unsigned const MachineRegisterInfo * MRI
static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder, CCAssignFn *AssignFn)
Helper function to compute forwarded registers for musttail calls.
static LLT getStackValueStoreTypeHack(const CCValAssign &VA)
static const uint32_t * getMaskForArgs(SmallVectorImpl< AArch64CallLowering::ArgInfo > &OutArgs, AArch64CallLowering::CallLoweringInfo &Info, const AArch64RegisterInfo &TRI, MachineFunction &MF)
static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT, MVT &LocVT)
static std::pair< CCAssignFn *, CCAssignFn * > getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI)
Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for CC.
static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect, bool IsTailCall)
static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt)
This file describes how to lower LLVM calls to machine code calls.
#define Success
static const MCPhysReg GPRArgRegs[]
static const MCPhysReg FPRArgRegs[]
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
This file defines ARC utility functions which are used by various parts of the compiler.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
@ Flags
Definition: TextStubV5.cpp:93
bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const override
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
bool canLowerReturn(MachineFunction &MF, CallingConv::ID CallConv, SmallVectorImpl< BaseArgInfo > &Outs, bool IsVarArg) const override
This hook must be implemented to check whether the return values described by Outs can fit into the r...
bool fallBackToDAGISel(const MachineFunction &MF) const override
bool isTypeIsValidForThisReturn(EVT Ty) const override
For targets which support the "returned" parameter attribute, returns true if the given type is a val...
bool isEligibleForTailCallOptimization(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info, SmallVectorImpl< ArgInfo > &InArgs, SmallVectorImpl< ArgInfo > &OutArgs) const
Returns true if the call can be lowered as a tail call.
AArch64CallLowering(const AArch64TargetLowering &TLI)
bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const override
This hook must be implemented to lower the given call instruction, including argument and return valu...
bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const override
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
bool isCallingConvWin64(CallingConv::ID CC) const
const RegisterBankInfo * getRegBankInfo() const override
bool hasCustomCallingConv() const
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC) const
Selects the correct CCAssignFn for a given CallingConvention value.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Selects the correct CCAssignFn for a given CallingConvention value.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:158
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void analyzeMustTailForwardedRegisters(SmallVectorImpl< ForwardedRegister > &Forwards, ArrayRef< MVT > RegParmTypes, CCAssignFn Fn)
Compute the set of registers that need to be preserved and forwarded to any musttail calls.
CallingConv::ID getCallingConv() const
unsigned getNextStackOffset() const
getNextStackOffset - Return the next stack offset such that all stack slots satisfy their alignment r...
bool isVarArg() const
bool isAllocated(MCRegister Reg) const
isAllocated - Return true if the specified register (or an alias) is allocated.
CCValAssign - Represent assignment of one arg/retval to a location.
LocInfo getLocInfo() const
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
bool handleAssignments(ValueHandler &Handler, SmallVectorImpl< ArgInfo > &Args, CCState &CCState, SmallVectorImpl< CCValAssign > &ArgLocs, MachineIRBuilder &MIRBuilder, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Use Handler to insert code to handle the argument/return values represented by Args.
void insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg, int FI) const
Load the returned value from the stack into virtual registers in VRegs.
bool determineAndHandleAssignments(ValueHandler &Handler, ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, bool IsVarArg, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Invoke ValueAssigner::assignArg on each of the given Args and then use Handler to move them to the as...
bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl< ArgInfo > &InArgs, ValueAssigner &CalleeAssigner, ValueAssigner &CallerAssigner) const
void splitToValueTypes(const ArgInfo &OrigArgInfo, SmallVectorImpl< ArgInfo > &SplitArgs, const DataLayout &DL, CallingConv::ID CallConv, SmallVectorImpl< uint64_t > *Offsets=nullptr) const
Break OrigArgInfo into one or more pieces the calling convention can process, returned in SplitArgs.
void insertSRetIncomingArgument(const Function &F, SmallVectorImpl< ArgInfo > &SplitArgs, Register &DemoteReg, MachineRegisterInfo &MRI, const DataLayout &DL) const
Insert the hidden sret ArgInfo to the beginning of SplitArgs.
void insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg) const
Store the return value given by VRegs into stack starting at the offset specified in DemoteReg.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< ArgInfo > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
bool determineAssignments(ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, CCState &CCInfo) const
Analyze the argument list in Args, using Assigner to populate CCInfo.
bool checkReturn(CCState &CCInfo, SmallVectorImpl< BaseArgInfo > &Outs, CCAssignFn *Fn) const
void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
iterator_range< arg_iterator > args()
Definition: Function.h:795
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:237
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:319
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:524
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:24
Machine Value Type.
bool isVector() const
Return true if this is a vector value type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:573
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasTailCall(bool V=true)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
Helper class to build MachineInstr.
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, unsigned Size)
Build and insert Res = G_ASSERT_ZEXT Op, Size.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StackOffset holds a fixed and a scalable offset in bytes.
Definition: TypeSize.h:36
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
const Triple & getTargetTriple() const
TargetOptions Options
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
virtual const RegisterBankInfo * getRegBankInfo() const
If the information for the register banks is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static IntegerType * getInt8Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
const ArrayRef< MCPhysReg > getFPRArgRegs()
const ArrayRef< MCPhysReg > getGPRArgRegs()
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ Swift
Calling convention for Swift.
Definition: CallingConv.h:69
@ PreserveMost
Used for runtime calls that preserves most registers.
Definition: CallingConv.h:63
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition: CallingConv.h:76
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
Definition: CallingConv.h:87
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Implicit
Not emitted register (e.g. carry, or temporary result).
std::optional< Function * > getAttachedARCFunction(const CallBase *CB)
This function returns operand bundle clang_arc_attachedcall's argument, which is the address of the A...
Definition: ObjCARCUtil.h:43
bool hasAttachedCallOpBundle(const CallBase *CB)
Definition: ObjCARCUtil.h:29
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
Definition: Utils.cpp:53
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1826
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:121
Align inferAlignFromPtrInfo(MachineFunction &MF, const MachinePointerInfo &MPO)
Definition: Utils.cpp:712
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
SmallVector< Register, 4 > Regs
Definition: CallLowering.h:63
SmallVector< ISD::ArgFlagsTy, 4 > Flags
Definition: CallLowering.h:51
Base class for ValueHandlers used for arguments coming into the current function, or for return value...
Definition: CallLowering.h:318
void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign VA) override
Provides a default implementation for argument handling.
Base class for ValueHandlers used for arguments passed to a function call, or for return values.
Definition: CallLowering.h:333
MachineRegisterInfo & MRI
Definition: CallLowering.h:231
virtual LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const
Return the in-memory size to write for the argument at VA.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:194
Describes a register that needs to be forwarded from the prologue to a musttail call.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.