Line data Source code
1 : //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file contains the AArch64 implementation of TargetFrameLowering class.
11 : //
12 : // On AArch64, stack frames are structured as follows:
13 : //
14 : // The stack grows downward.
15 : //
16 : // All of the individual frame areas on the frame below are optional, i.e. it's
17 : // possible to create a function so that the particular area isn't present
18 : // in the frame.
19 : //
20 : // At function entry, the "frame" looks as follows:
21 : //
22 : // | | Higher address
23 : // |-----------------------------------|
24 : // | |
25 : // | arguments passed on the stack |
26 : // | |
27 : // |-----------------------------------| <- sp
28 : // | | Lower address
29 : //
30 : //
31 : // After the prologue has run, the frame has the following general structure.
32 : // Note that this doesn't depict the case where a red-zone is used. Also,
33 : // technically the last frame area (VLAs) doesn't get created until in the
34 : // main function body, after the prologue is run. However, it's depicted here
35 : // for completeness.
36 : //
37 : // | | Higher address
38 : // |-----------------------------------|
39 : // | |
40 : // | arguments passed on the stack |
41 : // | |
42 : // |-----------------------------------|
43 : // | |
44 : // | (Win64 only) varargs from reg |
45 : // | |
46 : // |-----------------------------------|
47 : // | |
48 : // | prev_fp, prev_lr |
49 : // | (a.k.a. "frame record") |
50 : // |-----------------------------------| <- fp(=x29)
51 : // | |
52 : // | other callee-saved registers |
53 : // | |
54 : // |-----------------------------------|
55 : // |.empty.space.to.make.part.below....|
56 : // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
57 : // |.the.standard.16-byte.alignment....| compile time; if present)
58 : // |-----------------------------------|
59 : // | |
60 : // | local variables of fixed size |
61 : // | including spill slots |
62 : // |-----------------------------------| <- bp(not defined by ABI,
63 : // |.variable-sized.local.variables....| LLVM chooses X19)
64 : // |.(VLAs)............................| (size of this area is unknown at
65 : // |...................................| compile time)
66 : // |-----------------------------------| <- sp
67 : // | | Lower address
68 : //
69 : //
70 : // To access the data in a frame, at-compile time, a constant offset must be
71 : // computable from one of the pointers (fp, bp, sp) to access it. The size
72 : // of the areas with a dotted background cannot be computed at compile-time
73 : // if they are present, making it required to have all three of fp, bp and
74 : // sp to be set up to be able to access all contents in the frame areas,
75 : // assuming all of the frame areas are non-empty.
76 : //
77 : // For most functions, some of the frame areas are empty. For those functions,
78 : // it may not be necessary to set up fp or bp:
79 : // * A base pointer is definitely needed when there are both VLAs and local
80 : // variables with more-than-default alignment requirements.
81 : // * A frame pointer is definitely needed when there are local variables with
82 : // more-than-default alignment requirements.
83 : //
84 : // In some cases when a base pointer is not strictly needed, it is generated
85 : // anyway when offsets from the frame pointer to access local variables become
86 : // so large that the offset can't be encoded in the immediate fields of loads
87 : // or stores.
88 : //
89 : // FIXME: also explain the redzone concept.
90 : // FIXME: also explain the concept of reserved call frames.
91 : //
92 : //===----------------------------------------------------------------------===//
93 :
94 : #include "AArch64FrameLowering.h"
95 : #include "AArch64InstrInfo.h"
96 : #include "AArch64MachineFunctionInfo.h"
97 : #include "AArch64RegisterInfo.h"
98 : #include "AArch64Subtarget.h"
99 : #include "AArch64TargetMachine.h"
100 : #include "MCTargetDesc/AArch64AddressingModes.h"
101 : #include "llvm/ADT/ScopeExit.h"
102 : #include "llvm/ADT/SmallVector.h"
103 : #include "llvm/ADT/Statistic.h"
104 : #include "llvm/CodeGen/LivePhysRegs.h"
105 : #include "llvm/CodeGen/MachineBasicBlock.h"
106 : #include "llvm/CodeGen/MachineFrameInfo.h"
107 : #include "llvm/CodeGen/MachineFunction.h"
108 : #include "llvm/CodeGen/MachineInstr.h"
109 : #include "llvm/CodeGen/MachineInstrBuilder.h"
110 : #include "llvm/CodeGen/MachineMemOperand.h"
111 : #include "llvm/CodeGen/MachineModuleInfo.h"
112 : #include "llvm/CodeGen/MachineOperand.h"
113 : #include "llvm/CodeGen/MachineRegisterInfo.h"
114 : #include "llvm/CodeGen/RegisterScavenging.h"
115 : #include "llvm/CodeGen/TargetInstrInfo.h"
116 : #include "llvm/CodeGen/TargetRegisterInfo.h"
117 : #include "llvm/CodeGen/TargetSubtargetInfo.h"
118 : #include "llvm/IR/Attributes.h"
119 : #include "llvm/IR/CallingConv.h"
120 : #include "llvm/IR/DataLayout.h"
121 : #include "llvm/IR/DebugLoc.h"
122 : #include "llvm/IR/Function.h"
123 : #include "llvm/MC/MCDwarf.h"
124 : #include "llvm/Support/CommandLine.h"
125 : #include "llvm/Support/Debug.h"
126 : #include "llvm/Support/ErrorHandling.h"
127 : #include "llvm/Support/MathExtras.h"
128 : #include "llvm/Support/raw_ostream.h"
129 : #include "llvm/Target/TargetMachine.h"
130 : #include "llvm/Target/TargetOptions.h"
131 : #include <cassert>
132 : #include <cstdint>
133 : #include <iterator>
134 : #include <vector>
135 :
136 : using namespace llvm;
137 :
138 : #define DEBUG_TYPE "frame-info"
139 :
140 : static cl::opt<bool> EnableRedZone("aarch64-redzone",
141 : cl::desc("enable use of redzone on AArch64"),
142 : cl::init(false), cl::Hidden);
143 :
144 : static cl::opt<bool>
145 : ReverseCSRRestoreSeq("reverse-csr-restore-seq",
146 : cl::desc("reverse the CSR restore sequence"),
147 : cl::init(false), cl::Hidden);
148 :
149 : STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
150 :
151 : /// This is the biggest offset to the stack pointer we can encode in aarch64
152 : /// instructions (without using a separate calculation and a temp register).
153 : /// Note that the exception here are vector stores/loads which cannot encode any
154 : /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
155 : static const unsigned DefaultSafeSPDisplacement = 255;
156 :
157 : /// Look at each instruction that references stack frames and return the stack
158 : /// size limit beyond which some of these instructions will require a scratch
159 : /// register during their expansion later.
160 15047 : static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
161 : // FIXME: For now, just conservatively guestimate based on unscaled indexing
162 : // range. We'll end up allocating an unnecessary spill slot a lot, but
163 : // realistically that's not a big deal at this stage of the game.
164 33032 : for (MachineBasicBlock &MBB : MF) {
165 95161 : for (MachineInstr &MI : MBB) {
166 77077 : if (MI.isDebugInstr() || MI.isPseudo() ||
167 69702 : MI.getOpcode() == AArch64::ADDXri ||
168 : MI.getOpcode() == AArch64::ADDSXri)
169 8301 : continue;
170 :
171 256971 : for (const MachineOperand &MO : MI.operands()) {
172 188104 : if (!MO.isFI())
173 184407 : continue;
174 :
175 3697 : int Offset = 0;
176 3697 : if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
177 : AArch64FrameOffsetCannotUpdate)
178 8 : return 0;
179 : }
180 : }
181 : }
182 : return DefaultSafeSPDisplacement;
183 : }
184 :
185 19214 : bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
186 19214 : if (!EnableRedZone)
187 : return false;
188 : // Don't use the red zone if the function explicitly asks us not to.
189 : // This is typically used for kernel code.
190 94 : if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
191 : return false;
192 :
193 94 : const MachineFrameInfo &MFI = MF.getFrameInfo();
194 : const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
195 94 : unsigned NumBytes = AFI->getLocalStackSize();
196 :
197 94 : return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
198 : }
199 :
200 : /// hasFP - Return true if the specified function should have a dedicated frame
201 : /// pointer register.
202 294057 : bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
203 294057 : const MachineFrameInfo &MFI = MF.getFrameInfo();
204 294057 : const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
205 : // Retain behavior of always omitting the FP for leaf functions when possible.
206 294057 : if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
207 : return true;
208 288772 : if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
209 865908 : MFI.hasStackMap() || MFI.hasPatchPoint() ||
210 287253 : RegInfo->needsStackRealignment(MF))
211 3199 : return true;
212 : // With large callframes around we may need to use FP to access the scavenging
213 : // emergency spillslot.
214 : //
215 : // Unfortunately some calls to hasFP() like machine verifier ->
216 : // getReservedReg() -> hasFP in the middle of global isel are too early
217 : // to know the max call frame size. Hopefully conservatively returning "true"
218 : // in those cases is fine.
219 : // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
220 286800 : if (!MFI.isMaxCallFrameSizeComputed() ||
221 : MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
222 18565 : return true;
223 :
224 : return false;
225 : }
226 :
227 : /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
228 : /// not required, we reserve argument space for call sites in the function
229 : /// immediately on entry to the current function. This eliminates the need for
230 : /// add/sub sp brackets around call sites. Returns true if the call frame is
231 : /// included as part of the stack frame.
232 : bool
233 10297 : AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
234 10297 : return !MF.getFrameInfo().hasVarSizedObjects();
235 : }
236 :
237 3906 : MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
238 : MachineFunction &MF, MachineBasicBlock &MBB,
239 : MachineBasicBlock::iterator I) const {
240 : const AArch64InstrInfo *TII =
241 3906 : static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
242 : DebugLoc DL = I->getDebugLoc();
243 3906 : unsigned Opc = I->getOpcode();
244 3906 : bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
245 3906 : uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
246 :
247 3906 : const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
248 3906 : if (!TFI->hasReservedCallFrame(MF)) {
249 136 : unsigned Align = getStackAlignment();
250 :
251 136 : int64_t Amount = I->getOperand(0).getImm();
252 136 : Amount = alignTo(Amount, Align);
253 136 : if (!IsDestroy)
254 68 : Amount = -Amount;
255 :
256 : // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
257 : // doesn't have to pop anything), then the first operand will be zero too so
258 : // this adjustment is a no-op.
259 136 : if (CalleePopAmount == 0) {
260 : // FIXME: in-function stack adjustment for calls is limited to 24-bits
261 : // because there's no guaranteed temporary register available.
262 : //
263 : // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
264 : // 1) For offset <= 12-bit, we use LSL #0
265 : // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
266 : // LSL #0, and the other uses LSL #12.
267 : //
268 : // Most call frames will be allocated at the start of a function so
269 : // this is OK, but it is a limitation that needs dealing with.
270 : assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
271 135 : emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
272 : }
273 3770 : } else if (CalleePopAmount != 0) {
274 : // If the calling convention demands that the callee pops arguments from the
275 : // stack, we want to add it back if we have a reserved call frame.
276 : assert(CalleePopAmount < 0xffffff && "call frame too large");
277 12 : emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
278 : TII);
279 : }
280 3906 : return MBB.erase(I);
281 : }
282 :
283 29816 : static bool ShouldSignReturnAddress(MachineFunction &MF) {
284 : // The function should be signed in the following situations:
285 : // - sign-return-address=all
286 : // - sign-return-address=non-leaf and the functions spills the LR
287 :
288 29816 : const Function &F = MF.getFunction();
289 29816 : if (!F.hasFnAttribute("sign-return-address"))
290 : return false;
291 :
292 16 : StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
293 : if (Scope.equals("none"))
294 : return false;
295 :
296 : if (Scope.equals("all"))
297 : return true;
298 :
299 : assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
300 :
301 6 : for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
302 4 : if (Info.getReg() == AArch64::LR)
303 : return true;
304 :
305 : return false;
306 : }
307 :
308 1054 : void AArch64FrameLowering::emitCalleeSavedFrameMoves(
309 : MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
310 1054 : MachineFunction &MF = *MBB.getParent();
311 1054 : MachineFrameInfo &MFI = MF.getFrameInfo();
312 1054 : const TargetSubtargetInfo &STI = MF.getSubtarget();
313 1054 : const MCRegisterInfo *MRI = STI.getRegisterInfo();
314 1054 : const TargetInstrInfo *TII = STI.getInstrInfo();
315 : DebugLoc DL = MBB.findDebugLoc(MBBI);
316 :
317 : // Add callee saved registers to move list.
318 : const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
319 1054 : if (CSI.empty())
320 : return;
321 :
322 3854 : for (const auto &Info : CSI) {
323 2800 : unsigned Reg = Info.getReg();
324 : int64_t Offset =
325 2800 : MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
326 2800 : unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
327 : unsigned CFIIndex = MF.addFrameInst(
328 2800 : MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
329 5600 : BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
330 : .addCFIIndex(CFIIndex)
331 : .setMIFlags(MachineInstr::FrameSetup);
332 : }
333 : }
334 :
335 : // Find a scratch register that we can use at the start of the prologue to
336 : // re-align the stack pointer. We avoid using callee-save registers since they
337 : // may appear to be free when this is called from canUseAsPrologue (during
338 : // shrink wrapping), but then no longer be free when this is called from
339 : // emitPrologue.
340 : //
341 : // FIXME: This is a bit conservative, since in the above case we could use one
342 : // of the callee-save registers as a scratch temp to re-align the stack pointer,
343 : // but we would then have to make sure that we were in fact saving at least one
344 : // callee-save register in the prologue, which is additional complexity that
345 : // doesn't seem worth the benefit.
346 27 : static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
347 27 : MachineFunction *MF = MBB->getParent();
348 :
349 : // If MBB is an entry block, use X9 as the scratch register
350 27 : if (&MF->front() == MBB)
351 : return AArch64::X9;
352 :
353 9 : const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
354 : const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
355 9 : LivePhysRegs LiveRegs(TRI);
356 9 : LiveRegs.addLiveIns(*MBB);
357 :
358 : // Mark callee saved registers as used so we will not choose them.
359 9 : const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
360 189 : for (unsigned i = 0; CSRegs[i]; ++i)
361 180 : LiveRegs.addReg(CSRegs[i]);
362 :
363 : // Prefer X9 since it was historically used for the prologue scratch reg.
364 9 : const MachineRegisterInfo &MRI = MF->getRegInfo();
365 9 : if (LiveRegs.available(MRI, AArch64::X9))
366 : return AArch64::X9;
367 :
368 37 : for (unsigned Reg : AArch64::GPR64RegClass) {
369 36 : if (LiveRegs.available(MRI, Reg))
370 2 : return Reg;
371 : }
372 : return AArch64::NoRegister;
373 : }
374 :
375 78 : bool AArch64FrameLowering::canUseAsPrologue(
376 : const MachineBasicBlock &MBB) const {
377 78 : const MachineFunction *MF = MBB.getParent();
378 : MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
379 78 : const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
380 : const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
381 :
382 : // Don't need a scratch register if we're not going to re-align the stack.
383 78 : if (!RegInfo->needsStackRealignment(*MF))
384 : return true;
385 : // Otherwise, we can use any block as long as it has a scratch register
386 : // available.
387 7 : return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
388 : }
389 :
390 30858 : static bool windowsRequiresStackProbe(MachineFunction &MF,
391 : unsigned StackSizeInBytes) {
392 30858 : const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
393 30858 : if (!Subtarget.isTargetWindows())
394 : return false;
395 142 : const Function &F = MF.getFunction();
396 : // TODO: When implementing stack protectors, take that into account
397 : // for the probe threshold.
398 142 : unsigned StackProbeSize = 4096;
399 142 : if (F.hasFnAttribute("stack-probe-size"))
400 0 : F.getFnAttribute("stack-probe-size")
401 0 : .getValueAsString()
402 0 : .getAsInteger(0, StackProbeSize);
403 142 : return (StackSizeInBytes >= StackProbeSize) &&
404 6 : !F.hasFnAttribute("no-stack-arg-probe");
405 : }
406 :
407 16387 : bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
408 : MachineFunction &MF, unsigned StackBumpBytes) const {
409 16387 : AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
410 16387 : const MachineFrameInfo &MFI = MF.getFrameInfo();
411 16387 : const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
412 : const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
413 :
414 16387 : if (AFI->getLocalStackSize() == 0)
415 : return false;
416 :
417 : // 512 is the maximum immediate for stp/ldp that will be used for
418 : // callee-save save/restores
419 1373 : if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
420 50 : return false;
421 :
422 1323 : if (MFI.hasVarSizedObjects())
423 : return false;
424 :
425 1289 : if (RegInfo->needsStackRealignment(MF))
426 : return false;
427 :
428 : // This isn't strictly necessary, but it simplifies things a bit since the
429 : // current RedZone handling code assumes the SP is adjusted by the
430 : // callee-save save/restore code.
431 1265 : if (canUseRedZone(MF))
432 5 : return false;
433 :
434 : return true;
435 : }
436 :
437 : // Convert callee-save register save/restore instruction to do stack pointer
438 : // decrement/increment to allocate/deallocate the callee-save stack area by
439 : // converting store/load to use pre/post increment version.
440 1698 : static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
441 : MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
442 : const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc) {
443 : // Ignore instructions that do not operate on SP, i.e. shadow call stack
444 : // instructions.
445 3400 : while (MBBI->getOpcode() == AArch64::STRXpost ||
446 : MBBI->getOpcode() == AArch64::LDRXpre) {
447 : assert(MBBI->getOperand(0).getReg() != AArch64::SP);
448 : ++MBBI;
449 : }
450 :
451 : unsigned NewOpc;
452 : int Scale = 1;
453 1698 : switch (MBBI->getOpcode()) {
454 0 : default:
455 0 : llvm_unreachable("Unexpected callee-save save/restore opcode!");
456 : case AArch64::STPXi:
457 : NewOpc = AArch64::STPXpre;
458 : Scale = 8;
459 : break;
460 39 : case AArch64::STPDi:
461 : NewOpc = AArch64::STPDpre;
462 : Scale = 8;
463 39 : break;
464 4 : case AArch64::STPQi:
465 : NewOpc = AArch64::STPQpre;
466 : Scale = 16;
467 4 : break;
468 341 : case AArch64::STRXui:
469 : NewOpc = AArch64::STRXpre;
470 341 : break;
471 43 : case AArch64::STRDui:
472 : NewOpc = AArch64::STRDpre;
473 43 : break;
474 0 : case AArch64::STRQui:
475 : NewOpc = AArch64::STRQpre;
476 0 : break;
477 414 : case AArch64::LDPXi:
478 : NewOpc = AArch64::LDPXpost;
479 : Scale = 8;
480 414 : break;
481 39 : case AArch64::LDPDi:
482 : NewOpc = AArch64::LDPDpost;
483 : Scale = 8;
484 39 : break;
485 0 : case AArch64::LDPQi:
486 : NewOpc = AArch64::LDPQpost;
487 : Scale = 16;
488 0 : break;
489 351 : case AArch64::LDRXui:
490 : NewOpc = AArch64::LDRXpost;
491 351 : break;
492 43 : case AArch64::LDRDui:
493 : NewOpc = AArch64::LDRDpost;
494 43 : break;
495 0 : case AArch64::LDRQui:
496 : NewOpc = AArch64::LDRQpost;
497 0 : break;
498 : }
499 :
500 3396 : MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
501 1698 : MIB.addReg(AArch64::SP, RegState::Define);
502 :
503 : // Copy all operands other than the immediate offset.
504 : unsigned OpndIdx = 0;
505 6014 : for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
506 : ++OpndIdx)
507 4316 : MIB.add(MBBI->getOperand(OpndIdx));
508 :
509 : assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
510 : "Unexpected immediate offset in first/last callee-save save/restore "
511 : "instruction!");
512 : assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
513 : "Unexpected base register in callee-save save/restore instruction!");
514 : assert(CSStackSizeInc % Scale == 0);
515 1698 : MIB.addImm(CSStackSizeInc / Scale);
516 :
517 1698 : MIB.setMIFlags(MBBI->getFlags());
518 1698 : MIB.setMemRefs(MBBI->memoperands());
519 :
520 1698 : return std::prev(MBB.erase(MBBI));
521 : }
522 :
523 : // Fixup callee-save register save/restore instructions to take into account
524 : // combined SP bump by adding the local stack size to the stack offsets.
525 2036 : static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
526 : unsigned LocalStackSize) {
527 2036 : unsigned Opc = MI.getOpcode();
528 :
529 : // Ignore instructions that do not operate on SP, i.e. shadow call stack
530 : // instructions.
531 2036 : if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre) {
532 : assert(MI.getOperand(0).getReg() != AArch64::SP);
533 : return;
534 : }
535 :
536 : unsigned Scale;
537 2036 : switch (Opc) {
538 : case AArch64::STPXi:
539 : case AArch64::STRXui:
540 : case AArch64::STPDi:
541 : case AArch64::STRDui:
542 : case AArch64::LDPXi:
543 : case AArch64::LDRXui:
544 : case AArch64::LDPDi:
545 : case AArch64::LDRDui:
546 : Scale = 8;
547 : break;
548 8 : case AArch64::STPQi:
549 : case AArch64::STRQui:
550 : case AArch64::LDPQi:
551 : case AArch64::LDRQui:
552 : Scale = 16;
553 8 : break;
554 0 : default:
555 0 : llvm_unreachable("Unexpected callee-save save/restore opcode!");
556 : }
557 :
558 2036 : unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
559 : assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
560 : "Unexpected base register in callee-save save/restore instruction!");
561 : // Last operand is immediate offset that needs fixing.
562 2036 : MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
563 : // All generated opcodes have scaled offsets.
564 : assert(LocalStackSize % Scale == 0);
565 2036 : OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
566 : }
567 :
568 : static void adaptForLdStOpt(MachineBasicBlock &MBB,
569 : MachineBasicBlock::iterator FirstSPPopI,
570 : MachineBasicBlock::iterator LastPopI) {
571 : // Sometimes (when we restore in the same order as we save), we can end up
572 : // with code like this:
573 : //
574 : // ldp x26, x25, [sp]
575 : // ldp x24, x23, [sp, #16]
576 : // ldp x22, x21, [sp, #32]
577 : // ldp x20, x19, [sp, #48]
578 : // add sp, sp, #64
579 : //
580 : // In this case, it is always better to put the first ldp at the end, so
581 : // that the load-store optimizer can run and merge the ldp and the add into
582 : // a post-index ldp.
583 : // If we managed to grab the first pop instruction, move it to the end.
584 14036 : if (ReverseCSRRestoreSeq)
585 4 : MBB.splice(FirstSPPopI, &MBB, LastPopI);
586 : // We should end up with something like this now:
587 : //
588 : // ldp x24, x23, [sp, #16]
589 : // ldp x22, x21, [sp, #32]
590 : // ldp x20, x19, [sp, #48]
591 : // ldp x26, x25, [sp]
592 : // add sp, sp, #64
593 : //
594 : // and the load-store optimizer can merge the last two instructions into:
595 : //
596 : // ldp x26, x25, [sp], #64
597 : //
598 : }
599 :
600 14794 : void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
601 : MachineBasicBlock &MBB) const {
602 : MachineBasicBlock::iterator MBBI = MBB.begin();
603 14794 : const MachineFrameInfo &MFI = MF.getFrameInfo();
604 14794 : const Function &F = MF.getFunction();
605 14794 : const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
606 : const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
607 : const TargetInstrInfo *TII = Subtarget.getInstrInfo();
608 14794 : MachineModuleInfo &MMI = MF.getMMI();
609 14794 : AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
610 14794 : bool needsFrameMoves = MMI.hasDebugInfo() || F.needsUnwindTableEntry();
611 14794 : bool HasFP = hasFP(MF);
612 :
613 : // At this point, we're going to decide whether or not the function uses a
614 : // redzone. In most cases, the function doesn't have a redzone so let's
615 : // assume that's false and set it to true in the case that there's a redzone.
616 : AFI->setHasRedZone(false);
617 :
618 : // Debug location must be unknown since the first debug location is used
619 : // to determine the end of the prologue.
620 14794 : DebugLoc DL;
621 :
622 14794 : if (ShouldSignReturnAddress(MF)) {
623 12 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
624 : .setMIFlag(MachineInstr::FrameSetup);
625 : }
626 :
627 : // All calls are tail calls in GHC calling conv, and functions have no
628 : // prologue/epilogue.
629 29588 : if (MF.getFunction().getCallingConv() == CallingConv::GHC)
630 : return;
631 :
632 14788 : int NumBytes = (int)MFI.getStackSize();
633 14788 : if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
634 : assert(!HasFP && "unexpected function without stack frame but with FP");
635 :
636 : // All of the stack allocation is for locals.
637 : AFI->setLocalStackSize(NumBytes);
638 :
639 13423 : if (!NumBytes)
640 : return;
641 : // REDZONE: If the stack size is less than 128 bytes, we don't need
642 : // to actually allocate.
643 225 : if (canUseRedZone(MF)) {
644 : AFI->setHasRedZone(true);
645 : ++NumRedZoneFunctions;
646 : } else {
647 222 : emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
648 : MachineInstr::FrameSetup);
649 :
650 : // Label used to tie together the PROLOG_LABEL and the MachineMoves.
651 222 : MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
652 : // Encode the stack size of the leaf function.
653 : unsigned CFIIndex = MF.addFrameInst(
654 222 : MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
655 444 : BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
656 : .addCFIIndex(CFIIndex)
657 : .setMIFlags(MachineInstr::FrameSetup);
658 : }
659 225 : return;
660 : }
661 :
662 : bool IsWin64 =
663 1365 : Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
664 1299 : unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
665 :
666 1365 : auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
667 : // All of the remaining stack allocations are for locals.
668 1365 : AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
669 :
670 1365 : bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
671 1365 : if (CombineSPBump) {
672 514 : emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
673 : MachineInstr::FrameSetup);
674 : NumBytes = 0;
675 851 : } else if (PrologueSaveSize != 0) {
676 : MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(MBB, MBBI, DL, TII,
677 851 : -PrologueSaveSize);
678 851 : NumBytes -= PrologueSaveSize;
679 : }
680 : assert(NumBytes >= 0 && "Negative stack allocation size!?");
681 :
682 : // Move past the saves of the callee-saved registers, fixing up the offsets
683 : // and pre-inc if we decided to combine the callee-save and local stack
684 : // pointer bump above.
685 : MachineBasicBlock::iterator End = MBB.end();
686 3771 : while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
687 2406 : if (CombineSPBump)
688 1020 : fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize());
689 : ++MBBI;
690 : }
691 1365 : if (HasFP) {
692 : // Only set up FP if we actually need to. Frame pointer is fp =
693 : // sp - fixedobject - 16.
694 293 : int FPOffset = AFI->getCalleeSavedStackSize() - 16;
695 293 : if (CombineSPBump)
696 104 : FPOffset += AFI->getLocalStackSize();
697 :
698 : // Issue sub fp, sp, FPOffset or
699 : // mov fp,sp when FPOffset is zero.
700 : // Note: All stores of callee-saved registers are marked as "FrameSetup".
701 : // This code marks the instruction(s) that set the FP also.
702 293 : emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
703 : MachineInstr::FrameSetup);
704 : }
705 :
706 1365 : if (windowsRequiresStackProbe(MF, NumBytes)) {
707 2 : uint32_t NumWords = NumBytes >> 4;
708 :
709 4 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
710 2 : .addImm(NumWords)
711 : .setMIFlags(MachineInstr::FrameSetup);
712 :
713 2 : switch (MF.getTarget().getCodeModel()) {
714 1 : case CodeModel::Tiny:
715 : case CodeModel::Small:
716 : case CodeModel::Medium:
717 : case CodeModel::Kernel:
718 2 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
719 : .addExternalSymbol("__chkstk")
720 1 : .addReg(AArch64::X15, RegState::Implicit)
721 : .setMIFlags(MachineInstr::FrameSetup);
722 1 : break;
723 1 : case CodeModel::Large:
724 2 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
725 1 : .addReg(AArch64::X16, RegState::Define)
726 : .addExternalSymbol("__chkstk")
727 : .addExternalSymbol("__chkstk")
728 : .setMIFlags(MachineInstr::FrameSetup);
729 :
730 2 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
731 1 : .addReg(AArch64::X16, RegState::Kill)
732 1 : .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
733 : .setMIFlags(MachineInstr::FrameSetup);
734 1 : break;
735 : }
736 :
737 6 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
738 2 : .addReg(AArch64::SP, RegState::Kill)
739 2 : .addReg(AArch64::X15, RegState::Kill)
740 : .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
741 : .setMIFlags(MachineInstr::FrameSetup);
742 : NumBytes = 0;
743 : }
744 :
745 : // Allocate space for the rest of the frame.
746 1363 : if (NumBytes) {
747 54 : const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
748 : unsigned scratchSPReg = AArch64::SP;
749 :
750 54 : if (NeedsRealignment) {
751 20 : scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
752 : assert(scratchSPReg != AArch64::NoRegister);
753 : }
754 :
755 : // If we're a leaf function, try using the red zone.
756 54 : if (!canUseRedZone(MF))
757 : // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
758 : // the correct value here, as NumBytes also includes padding bytes,
759 : // which shouldn't be counted here.
760 53 : emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
761 : MachineInstr::FrameSetup);
762 :
763 54 : if (NeedsRealignment) {
764 20 : const unsigned Alignment = MFI.getMaxAlignment();
765 20 : const unsigned NrBitsToZero = countTrailingZeros(Alignment);
766 : assert(NrBitsToZero > 1);
767 : assert(scratchSPReg != AArch64::SP);
768 :
769 : // SUB X9, SP, NumBytes
770 : // -- X9 is temporary register, so shouldn't contain any live data here,
771 : // -- free to use. This is already produced by emitFrameOffset above.
772 : // AND SP, X9, 0b11111...0000
773 : // The logical immediates have a non-trivial encoding. The following
774 : // formula computes the encoded immediate with all ones but
775 : // NrBitsToZero zero bits as least significant bits.
776 20 : uint32_t andMaskEncoded = (1 << 12) // = N
777 20 : | ((64 - NrBitsToZero) << 6) // immr
778 20 : | ((64 - NrBitsToZero - 1) << 0); // imms
779 :
780 60 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
781 20 : .addReg(scratchSPReg, RegState::Kill)
782 20 : .addImm(andMaskEncoded);
783 : AFI->setStackRealigned(true);
784 : }
785 : }
786 :
787 : // If we need a base pointer, set it up here. It's whatever the value of the
788 : // stack pointer is at this point. Any variable size objects will be allocated
789 : // after this, so we can still use the base pointer to reference locals.
790 : //
791 : // FIXME: Clarify FrameSetup flags here.
792 : // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
793 : // needed.
794 1365 : if (RegInfo->hasBasePointer(MF)) {
795 15 : TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
796 15 : false);
797 : }
798 :
799 1365 : if (needsFrameMoves) {
800 1054 : const DataLayout &TD = MF.getDataLayout();
801 1054 : const int StackGrowth = -TD.getPointerSize(0);
802 1054 : unsigned FramePtr = RegInfo->getFrameRegister(MF);
803 : // An example of the prologue:
804 : //
805 : // .globl __foo
806 : // .align 2
807 : // __foo:
808 : // Ltmp0:
809 : // .cfi_startproc
810 : // .cfi_personality 155, ___gxx_personality_v0
811 : // Leh_func_begin:
812 : // .cfi_lsda 16, Lexception33
813 : //
814 : // stp xa,bx, [sp, -#offset]!
815 : // ...
816 : // stp x28, x27, [sp, #offset-32]
817 : // stp fp, lr, [sp, #offset-16]
818 : // add fp, sp, #offset - 16
819 : // sub sp, sp, #1360
820 : //
821 : // The Stack:
822 : // +-------------------------------------------+
823 : // 10000 | ........ | ........ | ........ | ........ |
824 : // 10004 | ........ | ........ | ........ | ........ |
825 : // +-------------------------------------------+
826 : // 10008 | ........ | ........ | ........ | ........ |
827 : // 1000c | ........ | ........ | ........ | ........ |
828 : // +===========================================+
829 : // 10010 | X28 Register |
830 : // 10014 | X28 Register |
831 : // +-------------------------------------------+
832 : // 10018 | X27 Register |
833 : // 1001c | X27 Register |
834 : // +===========================================+
835 : // 10020 | Frame Pointer |
836 : // 10024 | Frame Pointer |
837 : // +-------------------------------------------+
838 : // 10028 | Link Register |
839 : // 1002c | Link Register |
840 : // +===========================================+
841 : // 10030 | ........ | ........ | ........ | ........ |
842 : // 10034 | ........ | ........ | ........ | ........ |
843 : // +-------------------------------------------+
844 : // 10038 | ........ | ........ | ........ | ........ |
845 : // 1003c | ........ | ........ | ........ | ........ |
846 : // +-------------------------------------------+
847 : //
848 : // [sp] = 10030 :: >>initial value<<
849 : // sp = 10020 :: stp fp, lr, [sp, #-16]!
850 : // fp = sp == 10020 :: mov fp, sp
851 : // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
852 : // sp == 10010 :: >>final value<<
853 : //
854 : // The frame pointer (w29) points to address 10020. If we use an offset of
855 : // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
856 : // for w27, and -32 for w28:
857 : //
858 : // Ltmp1:
859 : // .cfi_def_cfa w29, 16
860 : // Ltmp2:
861 : // .cfi_offset w30, -8
862 : // Ltmp3:
863 : // .cfi_offset w29, -16
864 : // Ltmp4:
865 : // .cfi_offset w27, -24
866 : // Ltmp5:
867 : // .cfi_offset w28, -32
868 :
869 1054 : if (HasFP) {
870 : // Define the current CFA rule to use the provided FP.
871 186 : unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
872 372 : unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
873 186 : nullptr, Reg, 2 * StackGrowth - FixedObject));
874 372 : BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
875 : .addCFIIndex(CFIIndex)
876 : .setMIFlags(MachineInstr::FrameSetup);
877 : } else {
878 : // Encode the stack size of the leaf function.
879 : unsigned CFIIndex = MF.addFrameInst(
880 868 : MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize()));
881 1736 : BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
882 : .addCFIIndex(CFIIndex)
883 : .setMIFlags(MachineInstr::FrameSetup);
884 : }
885 :
886 : // Now emit the moves for whatever callee saved regs we have (including FP,
887 : // LR if those are saved).
888 1054 : emitCalleeSavedFrameMoves(MBB, MBBI);
889 : }
890 : }
891 :
892 15022 : static void InsertReturnAddressAuth(MachineFunction &MF,
893 : MachineBasicBlock &MBB) {
894 15022 : if (!ShouldSignReturnAddress(MF))
895 15016 : return;
896 6 : const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
897 : const TargetInstrInfo *TII = Subtarget.getInstrInfo();
898 :
899 6 : MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
900 6 : DebugLoc DL;
901 6 : if (MBBI != MBB.end())
902 : DL = MBBI->getDebugLoc();
903 :
904 : // The AUTIASP instruction assembles to a hint instruction before v8.3a so
905 : // this instruction can safely used for any v8a architecture.
906 : // From v8.3a onwards there are optimised authenticate LR and return
907 : // instructions, namely RETA{A,B}, that can be used instead.
908 6 : if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
909 1 : MBBI->getOpcode() == AArch64::RET_ReallyLR) {
910 2 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::RETAA)).copyImplicitOps(*MBBI);
911 1 : MBB.erase(MBBI);
912 : } else {
913 10 : BuildMI(MBB, MBBI, DL, TII->get(AArch64::AUTIASP))
914 : .setMIFlag(MachineInstr::FrameDestroy);
915 : }
916 : }
917 :
918 15028 : void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
919 : MachineBasicBlock &MBB) const {
920 15028 : MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
921 15028 : MachineFrameInfo &MFI = MF.getFrameInfo();
922 15028 : const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
923 : const TargetInstrInfo *TII = Subtarget.getInstrInfo();
924 15028 : DebugLoc DL;
925 : bool IsTailCallReturn = false;
926 15028 : if (MBB.end() != MBBI) {
927 : DL = MBBI->getDebugLoc();
928 15028 : unsigned RetOpcode = MBBI->getOpcode();
929 30056 : IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
930 : RetOpcode == AArch64::TCRETURNri ||
931 15028 : RetOpcode == AArch64::TCRETURNriBTI;
932 : }
933 15028 : int NumBytes = MFI.getStackSize();
934 15028 : const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
935 :
936 : // All calls are tail calls in GHC calling conv, and functions have no
937 : // prologue/epilogue.
938 30056 : if (MF.getFunction().getCallingConv() == CallingConv::GHC)
939 : return;
940 :
941 : // Initial and residual are named for consistency with the prologue. Note that
942 : // in the epilogue, the residual adjustment is executed first.
943 : uint64_t ArgumentPopSize = 0;
944 15022 : if (IsTailCallReturn) {
945 225 : MachineOperand &StackAdjust = MBBI->getOperand(1);
946 :
947 : // For a tail-call in a callee-pops-arguments environment, some or all of
948 : // the stack may actually be in use for the call's arguments, this is
949 : // calculated during LowerCall and consumed here...
950 225 : ArgumentPopSize = StackAdjust.getImm();
951 : } else {
952 : // ... otherwise the amount to pop is *all* of the argument space,
953 : // conveniently stored in the MachineFunctionInfo by
954 : // LowerFormalArguments. This will, of course, be zero for the C calling
955 : // convention.
956 14797 : ArgumentPopSize = AFI->getArgumentStackToRestore();
957 : }
958 :
959 : // The stack frame should be like below,
960 : //
961 : // ---------------------- ---
962 : // | | |
963 : // | BytesInStackArgArea| CalleeArgStackSize
964 : // | (NumReusableBytes) | (of tail call)
965 : // | | ---
966 : // | | |
967 : // ---------------------| --- |
968 : // | | | |
969 : // | CalleeSavedReg | | |
970 : // | (CalleeSavedStackSize)| | |
971 : // | | | |
972 : // ---------------------| | NumBytes
973 : // | | StackSize (StackAdjustUp)
974 : // | LocalStackSize | | |
975 : // | (covering callee | | |
976 : // | args) | | |
977 : // | | | |
978 : // ---------------------- --- ---
979 : //
980 : // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
981 : // = StackSize + ArgumentPopSize
982 : //
983 : // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
984 : // it as the 2nd argument of AArch64ISD::TC_RETURN.
985 :
986 15022 : auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
987 :
988 : bool IsWin64 =
989 : Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
990 14934 : unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
991 :
992 : uint64_t AfterCSRPopSize = ArgumentPopSize;
993 15022 : auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
994 15022 : bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
995 : // Assume we can't combine the last pop with the sp restore.
996 :
997 15022 : if (!CombineSPBump && PrologueSaveSize != 0) {
998 853 : MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
999 : // Converting the last ldp to a post-index ldp is valid only if the last
1000 : // ldp's offset is 0.
1001 853 : const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1002 : // If the offset is 0, convert it to a post-index ldp.
1003 853 : if (OffsetOp.getImm() == 0) {
1004 : convertCalleeSaveRestoreToSPPrePostIncDec(MBB, Pop, DL, TII,
1005 847 : PrologueSaveSize);
1006 : } else {
1007 : // If not, make sure to emit an add after the last ldp.
1008 : // We're doing this by transfering the size to be restored from the
1009 : // adjustment *before* the CSR pops to the adjustment *after* the CSR
1010 : // pops.
1011 6 : AfterCSRPopSize += PrologueSaveSize;
1012 : }
1013 : }
1014 :
1015 : // Move past the restores of the callee-saved registers.
1016 : // If we plan on combining the sp bump of the local stack size and the callee
1017 : // save stack size, we might need to adjust the CSR save and restore offsets.
1018 15022 : MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
1019 : MachineBasicBlock::iterator Begin = MBB.begin();
1020 17404 : while (LastPopI != Begin) {
1021 : --LastPopI;
1022 16875 : if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
1023 : ++LastPopI;
1024 : break;
1025 2382 : } else if (CombineSPBump)
1026 1016 : fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize());
1027 : }
1028 :
1029 : // If there is a single SP update, insert it before the ret and we're done.
1030 15022 : if (CombineSPBump) {
1031 746 : emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1032 746 : NumBytes + AfterCSRPopSize, TII,
1033 : MachineInstr::FrameDestroy);
1034 746 : return;
1035 : }
1036 :
1037 14276 : NumBytes -= PrologueSaveSize;
1038 : assert(NumBytes >= 0 && "Negative stack allocation size!?");
1039 :
1040 14276 : if (!hasFP(MF)) {
1041 14089 : bool RedZone = canUseRedZone(MF);
1042 : // If this was a redzone leaf function, we don't need to restore the
1043 : // stack pointer (but we may need to pop stack args for fastcc).
1044 14089 : if (RedZone && AfterCSRPopSize == 0)
1045 : return;
1046 :
1047 14036 : bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1048 14036 : int StackRestoreBytes = RedZone ? 0 : NumBytes;
1049 14036 : if (NoCalleeSaveRestore)
1050 13370 : StackRestoreBytes += AfterCSRPopSize;
1051 :
1052 : // If we were able to combine the local stack pop with the argument pop,
1053 : // then we're done.
1054 14036 : bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1055 :
1056 : // If we're done after this, make sure to help the load store optimizer.
1057 14036 : if (Done)
1058 28054 : adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1059 :
1060 14036 : emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1061 : StackRestoreBytes, TII, MachineInstr::FrameDestroy);
1062 14036 : if (Done)
1063 : return;
1064 :
1065 : NumBytes = 0;
1066 : }
1067 :
1068 : // Restore the original stack pointer.
1069 : // FIXME: Rather than doing the math here, we should instead just use
1070 : // non-post-indexed loads for the restores if we aren't actually going to
1071 : // be able to save any instructions.
1072 196 : if (MFI.hasVarSizedObjects() || AFI->isStackRealigned())
1073 54 : emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1074 54 : -AFI->getCalleeSavedStackSize() + 16, TII,
1075 : MachineInstr::FrameDestroy);
1076 142 : else if (NumBytes)
1077 7 : emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
1078 : MachineInstr::FrameDestroy);
1079 :
1080 : // This must be placed after the callee-save restore code because that code
1081 : // assumes the SP is at the same location as it was after the callee-save save
1082 : // code in the prologue.
1083 196 : if (AfterCSRPopSize) {
1084 : // Find an insertion point for the first ldp so that it goes before the
1085 : // shadow call stack epilog instruction. This ensures that the restore of
1086 : // lr from x18 is placed after the restore from sp.
1087 9 : auto FirstSPPopI = MBB.getFirstTerminator();
1088 11 : while (FirstSPPopI != Begin) {
1089 11 : auto Prev = std::prev(FirstSPPopI);
1090 22 : if (Prev->getOpcode() != AArch64::LDRXpre ||
1091 2 : Prev->getOperand(0).getReg() == AArch64::SP)
1092 : break;
1093 : FirstSPPopI = Prev;
1094 : }
1095 :
1096 9 : adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1097 :
1098 9 : emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1099 : AfterCSRPopSize, TII, MachineInstr::FrameDestroy);
1100 : }
1101 : }
1102 :
1103 : /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1104 : /// debug info. It's the same as what we use for resolving the code-gen
1105 : /// references for now. FIXME: This can go wrong when references are
1106 : /// SP-relative and simple call frames aren't used.
1107 15 : int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
1108 : int FI,
1109 : unsigned &FrameReg) const {
1110 15 : return resolveFrameIndexReference(MF, FI, FrameReg);
1111 : }
1112 :
1113 3947 : int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
1114 : int FI, unsigned &FrameReg,
1115 : bool PreferFP) const {
1116 3947 : const MachineFrameInfo &MFI = MF.getFrameInfo();
1117 : const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
1118 3947 : MF.getSubtarget().getRegisterInfo());
1119 : const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1120 3947 : const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1121 : bool IsWin64 =
1122 3947 : Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1123 3555 : unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1124 3947 : int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16;
1125 3947 : int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
1126 : bool isFixed = MFI.isFixedObjectIndex(FI);
1127 3575 : bool isCSR = !isFixed && MFI.getObjectOffset(FI) >=
1128 3575 : -((int)AFI->getCalleeSavedStackSize());
1129 :
1130 : // Use frame pointer to reference fixed objects. Use it for locals if
1131 : // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1132 : // reliable as a base). Make sure useFPForScavengingIndex() does the
1133 : // right thing for the emergency spill slot.
1134 : bool UseFP = false;
1135 3947 : if (AFI->hasStackFrame()) {
1136 : // Note: Keeping the following as multiple 'if' statements rather than
1137 : // merging to a single expression for readability.
1138 : //
1139 : // Argument access should always use the FP.
1140 3011 : if (isFixed) {
1141 194 : UseFP = hasFP(MF);
1142 2817 : } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1143 : // References to the CSR area must use FP if we're re-aligning the stack
1144 : // since the dynamically-sized alignment padding is between the SP/BP and
1145 : // the CSR area.
1146 : assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1147 : UseFP = true;
1148 2816 : } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1149 : // If the FPOffset is negative, we have to keep in mind that the
1150 : // available offset range for negative offsets is smaller than for
1151 : // positive ones. If an offset is
1152 : // available via the FP and the SP, use whichever is closest.
1153 476 : bool FPOffsetFits = FPOffset >= -256;
1154 476 : PreferFP |= Offset > -FPOffset;
1155 :
1156 476 : if (MFI.hasVarSizedObjects()) {
1157 : // If we have variable sized objects, we can use either FP or BP, as the
1158 : // SP offset is unknown. We can use the base pointer if we have one and
1159 : // FP is not preferred. If not, we're stuck with using FP.
1160 45 : bool CanUseBP = RegInfo->hasBasePointer(MF);
1161 45 : if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1162 : UseFP = PreferFP;
1163 40 : else if (!CanUseBP) // Can't use BP. Forced to use FP.
1164 : UseFP = true;
1165 : // else we can use BP and FP, but the offset from FP won't fit.
1166 : // That will make us scavenge registers which we can probably avoid by
1167 : // using BP. If it won't fit for BP either, we'll scavenge anyway.
1168 431 : } else if (FPOffset >= 0) {
1169 : // Use SP or FP, whichever gives us the best chance of the offset
1170 : // being in range for direct access. If the FPOffset is positive,
1171 : // that'll always be best, as the SP will be even further away.
1172 : UseFP = true;
1173 : } else {
1174 : // We have the choice between FP and (SP or BP).
1175 431 : if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1176 : UseFP = true;
1177 : }
1178 : }
1179 : }
1180 :
1181 : assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1182 : "In the presence of dynamic stack pointer realignment, "
1183 : "non-argument/CSR objects cannot be accessed through the frame pointer");
1184 :
1185 199 : if (UseFP) {
1186 355 : FrameReg = RegInfo->getFrameRegister(MF);
1187 355 : return FPOffset;
1188 : }
1189 :
1190 : // Use the base pointer if we have one.
1191 3592 : if (RegInfo->hasBasePointer(MF))
1192 11 : FrameReg = RegInfo->getBaseRegister();
1193 : else {
1194 : assert(!MFI.hasVarSizedObjects() &&
1195 : "Can't use SP when we have var sized objects.");
1196 3581 : FrameReg = AArch64::SP;
1197 : // If we're using the red zone for this function, the SP won't actually
1198 : // be adjusted, so the offsets will be negative. They're also all
1199 : // within range of the signed 9-bit immediate instructions.
1200 3581 : if (canUseRedZone(MF))
1201 23 : Offset -= AFI->getLocalStackSize();
1202 : }
1203 :
1204 : return Offset;
1205 : }
1206 :
1207 : static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1208 : // Do not set a kill flag on values that are also marked as live-in. This
1209 : // happens with the @llvm-returnaddress intrinsic and with arguments passed in
1210 : // callee saved registers.
1211 : // Omitting the kill flags is conservatively correct even if the live-in
1212 : // is not used after all.
1213 4033 : bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1214 : return getKillRegState(!IsLiveIn);
1215 : }
1216 :
1217 3967 : static bool produceCompactUnwindFrame(MachineFunction &MF) {
1218 3967 : const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1219 3967 : AttributeList Attrs = MF.getFunction().getAttributes();
1220 5620 : return Subtarget.isTargetMachO() &&
1221 3306 : !(Subtarget.getTargetLowering()->supportSwiftError() &&
1222 1653 : Attrs.hasAttrSomewhere(Attribute::SwiftError));
1223 : }
1224 :
1225 : namespace {
1226 :
1227 : struct RegPairInfo {
1228 : unsigned Reg1 = AArch64::NoRegister;
1229 : unsigned Reg2 = AArch64::NoRegister;
1230 : int FrameIdx;
1231 : int Offset;
1232 : enum RegType { GPR, FPR64, FPR128 } Type;
1233 :
1234 0 : RegPairInfo() = default;
1235 :
1236 0 : bool isPaired() const { return Reg2 != AArch64::NoRegister; }
1237 : };
1238 :
1239 : } // end anonymous namespace
1240 :
1241 0 : static void computeCalleeSaveRegisterPairs(
1242 : MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
1243 : const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
1244 : bool &NeedShadowCallStackProlog) {
1245 :
1246 0 : if (CSI.empty())
1247 0 : return;
1248 :
1249 0 : AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1250 0 : MachineFrameInfo &MFI = MF.getFrameInfo();
1251 : CallingConv::ID CC = MF.getFunction().getCallingConv();
1252 0 : unsigned Count = CSI.size();
1253 : (void)CC;
1254 : // MachO's compact unwind format relies on all registers being stored in
1255 : // pairs.
1256 : assert((!produceCompactUnwindFrame(MF) ||
1257 : CC == CallingConv::PreserveMost ||
1258 : (Count & 1) == 0) &&
1259 : "Odd number of callee-saved regs to spill!");
1260 0 : int Offset = AFI->getCalleeSavedStackSize();
1261 :
1262 0 : for (unsigned i = 0; i < Count; ++i) {
1263 : RegPairInfo RPI;
1264 0 : RPI.Reg1 = CSI[i].getReg();
1265 :
1266 0 : if (AArch64::GPR64RegClass.contains(RPI.Reg1))
1267 0 : RPI.Type = RegPairInfo::GPR;
1268 0 : else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
1269 0 : RPI.Type = RegPairInfo::FPR64;
1270 0 : else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
1271 0 : RPI.Type = RegPairInfo::FPR128;
1272 : else
1273 0 : llvm_unreachable("Unsupported register class.");
1274 :
1275 : // Add the next reg to the pair if it is in the same register class.
1276 0 : if (i + 1 < Count) {
1277 0 : unsigned NextReg = CSI[i + 1].getReg();
1278 0 : switch (RPI.Type) {
1279 0 : case RegPairInfo::GPR:
1280 0 : if (AArch64::GPR64RegClass.contains(NextReg))
1281 0 : RPI.Reg2 = NextReg;
1282 : break;
1283 0 : case RegPairInfo::FPR64:
1284 0 : if (AArch64::FPR64RegClass.contains(NextReg))
1285 0 : RPI.Reg2 = NextReg;
1286 : break;
1287 0 : case RegPairInfo::FPR128:
1288 0 : if (AArch64::FPR128RegClass.contains(NextReg))
1289 0 : RPI.Reg2 = NextReg;
1290 : break;
1291 : }
1292 : }
1293 :
1294 : // If either of the registers to be saved is the lr register, it means that
1295 : // we also need to save lr in the shadow call stack.
1296 0 : if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
1297 0 : MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
1298 0 : if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
1299 0 : report_fatal_error("Must reserve x18 to use shadow call stack");
1300 0 : NeedShadowCallStackProlog = true;
1301 : }
1302 :
1303 : // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
1304 : // list to come in sorted by frame index so that we can issue the store
1305 : // pair instructions directly. Assert if we see anything otherwise.
1306 : //
1307 : // The order of the registers in the list is controlled by
1308 : // getCalleeSavedRegs(), so they will always be in-order, as well.
1309 : assert((!RPI.isPaired() ||
1310 : (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
1311 : "Out of order callee saved regs!");
1312 :
1313 : // MachO's compact unwind format relies on all registers being stored in
1314 : // adjacent register pairs.
1315 : assert((!produceCompactUnwindFrame(MF) ||
1316 : CC == CallingConv::PreserveMost ||
1317 : (RPI.isPaired() &&
1318 : ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
1319 : RPI.Reg1 + 1 == RPI.Reg2))) &&
1320 : "Callee-save registers not saved as adjacent register pair!");
1321 :
1322 0 : RPI.FrameIdx = CSI[i].getFrameIdx();
1323 :
1324 0 : int Scale = RPI.Type == RegPairInfo::FPR128 ? 16 : 8;
1325 0 : Offset -= RPI.isPaired() ? 2 * Scale : Scale;
1326 :
1327 : // Round up size of non-pair to pair size if we need to pad the
1328 : // callee-save area to ensure 16-byte alignment.
1329 0 : if (AFI->hasCalleeSaveStackFreeSpace() &&
1330 0 : RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired()) {
1331 0 : Offset -= 8;
1332 : assert(Offset % 16 == 0);
1333 : assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
1334 : MFI.setObjectAlignment(RPI.FrameIdx, 16);
1335 : }
1336 :
1337 : assert(Offset % Scale == 0);
1338 0 : RPI.Offset = Offset / Scale;
1339 : assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
1340 : "Offset out of bounds for LDP/STP immediate");
1341 :
1342 0 : RegPairs.push_back(RPI);
1343 0 : if (RPI.isPaired())
1344 : ++i;
1345 : }
1346 : }
1347 :
1348 1365 : bool AArch64FrameLowering::spillCalleeSavedRegisters(
1349 : MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1350 : const std::vector<CalleeSavedInfo> &CSI,
1351 : const TargetRegisterInfo *TRI) const {
1352 1365 : MachineFunction &MF = *MBB.getParent();
1353 1365 : const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1354 1365 : DebugLoc DL;
1355 1365 : SmallVector<RegPairInfo, 8> RegPairs;
1356 :
1357 1365 : bool NeedShadowCallStackProlog = false;
1358 1365 : computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1359 : NeedShadowCallStackProlog);
1360 1365 : const MachineRegisterInfo &MRI = MF.getRegInfo();
1361 :
1362 1365 : if (NeedShadowCallStackProlog) {
1363 : // Shadow call stack prolog: str x30, [x18], #8
1364 6 : BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
1365 2 : .addReg(AArch64::X18, RegState::Define)
1366 2 : .addReg(AArch64::LR)
1367 2 : .addReg(AArch64::X18)
1368 : .addImm(8)
1369 : .setMIFlag(MachineInstr::FrameSetup);
1370 :
1371 : // This instruction also makes x18 live-in to the entry block.
1372 : MBB.addLiveIn(AArch64::X18);
1373 : }
1374 :
1375 3766 : for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
1376 : ++RPII) {
1377 2401 : RegPairInfo RPI = *RPII;
1378 : unsigned Reg1 = RPI.Reg1;
1379 : unsigned Reg2 = RPI.Reg2;
1380 : unsigned StrOpc;
1381 :
1382 : // Issue sequence of spills for cs regs. The first spill may be converted
1383 : // to a pre-decrement store later by emitPrologue if the callee-save stack
1384 : // area allocation can't be combined with the local stack area allocation.
1385 : // For example:
1386 : // stp x22, x21, [sp, #0] // addImm(+0)
1387 : // stp x20, x19, [sp, #16] // addImm(+2)
1388 : // stp fp, lr, [sp, #32] // addImm(+4)
1389 : // Rationale: This sequence saves uop updates compared to a sequence of
1390 : // pre-increment spills like stp xi,xj,[sp,#-16]!
1391 : // Note: Similar rationale and sequence for restores in epilog.
1392 : unsigned Size, Align;
1393 2401 : switch (RPI.Type) {
1394 2103 : case RegPairInfo::GPR:
1395 2103 : StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
1396 : Size = 8;
1397 : Align = 8;
1398 : break;
1399 279 : case RegPairInfo::FPR64:
1400 279 : StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
1401 : Size = 8;
1402 : Align = 8;
1403 : break;
1404 19 : case RegPairInfo::FPR128:
1405 19 : StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
1406 : Size = 16;
1407 : Align = 16;
1408 : break;
1409 : }
1410 : LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
1411 : if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1412 : dbgs() << ") -> fi#(" << RPI.FrameIdx;
1413 : if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1414 : dbgs() << ")\n");
1415 :
1416 4802 : MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
1417 2401 : if (!MRI.isReserved(Reg1))
1418 2386 : MBB.addLiveIn(Reg1);
1419 2401 : if (RPI.isPaired()) {
1420 1632 : if (!MRI.isReserved(Reg2))
1421 1153 : MBB.addLiveIn(Reg2);
1422 1632 : MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
1423 : MIB.addMemOperand(MF.getMachineMemOperand(
1424 : MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
1425 3264 : MachineMemOperand::MOStore, Size, Align));
1426 : }
1427 2401 : MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
1428 2401 : .addReg(AArch64::SP)
1429 2401 : .addImm(RPI.Offset) // [sp, #offset*scale],
1430 : // where factor*scale is implicit
1431 : .setMIFlag(MachineInstr::FrameSetup);
1432 : MIB.addMemOperand(MF.getMachineMemOperand(
1433 : MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
1434 4802 : MachineMemOperand::MOStore, Size, Align));
1435 : }
1436 1365 : return true;
1437 : }
1438 :
1439 1372 : bool AArch64FrameLowering::restoreCalleeSavedRegisters(
1440 : MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1441 : std::vector<CalleeSavedInfo> &CSI,
1442 : const TargetRegisterInfo *TRI) const {
1443 1372 : MachineFunction &MF = *MBB.getParent();
1444 1372 : const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1445 1372 : DebugLoc DL;
1446 1372 : SmallVector<RegPairInfo, 8> RegPairs;
1447 :
1448 1372 : if (MI != MBB.end())
1449 : DL = MI->getDebugLoc();
1450 :
1451 1372 : bool NeedShadowCallStackProlog = false;
1452 1372 : computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1453 : NeedShadowCallStackProlog);
1454 :
1455 : auto EmitMI = [&](const RegPairInfo &RPI) {
1456 : unsigned Reg1 = RPI.Reg1;
1457 : unsigned Reg2 = RPI.Reg2;
1458 :
1459 : // Issue sequence of restores for cs regs. The last restore may be converted
1460 : // to a post-increment load later by emitEpilogue if the callee-save stack
1461 : // area allocation can't be combined with the local stack area allocation.
1462 : // For example:
1463 : // ldp fp, lr, [sp, #32] // addImm(+4)
1464 : // ldp x20, x19, [sp, #16] // addImm(+2)
1465 : // ldp x22, x21, [sp, #0] // addImm(+0)
1466 : // Note: see comment in spillCalleeSavedRegisters()
1467 : unsigned LdrOpc;
1468 : unsigned Size, Align;
1469 : switch (RPI.Type) {
1470 : case RegPairInfo::GPR:
1471 : LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
1472 : Size = 8;
1473 : Align = 8;
1474 : break;
1475 : case RegPairInfo::FPR64:
1476 : LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
1477 : Size = 8;
1478 : Align = 8;
1479 : break;
1480 : case RegPairInfo::FPR128:
1481 : LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
1482 : Size = 16;
1483 : Align = 16;
1484 : break;
1485 : }
1486 : LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
1487 : if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1488 : dbgs() << ") -> fi#(" << RPI.FrameIdx;
1489 : if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1490 : dbgs() << ")\n");
1491 :
1492 : MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
1493 : if (RPI.isPaired()) {
1494 : MIB.addReg(Reg2, getDefRegState(true));
1495 : MIB.addMemOperand(MF.getMachineMemOperand(
1496 : MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
1497 : MachineMemOperand::MOLoad, Size, Align));
1498 : }
1499 : MIB.addReg(Reg1, getDefRegState(true))
1500 : .addReg(AArch64::SP)
1501 : .addImm(RPI.Offset) // [sp, #offset*scale]
1502 : // where factor*scale is implicit
1503 : .setMIFlag(MachineInstr::FrameDestroy);
1504 : MIB.addMemOperand(MF.getMachineMemOperand(
1505 : MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
1506 : MachineMemOperand::MOLoad, Size, Align));
1507 1372 : };
1508 :
1509 1372 : if (ReverseCSRRestoreSeq)
1510 26 : for (const RegPairInfo &RPI : reverse(RegPairs))
1511 20 : EmitMI(RPI);
1512 : else
1513 3726 : for (const RegPairInfo &RPI : RegPairs)
1514 2360 : EmitMI(RPI);
1515 :
1516 1372 : if (NeedShadowCallStackProlog) {
1517 : // Shadow call stack epilog: ldr x30, [x18, #-8]!
1518 6 : BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
1519 2 : .addReg(AArch64::X18, RegState::Define)
1520 2 : .addReg(AArch64::LR, RegState::Define)
1521 2 : .addReg(AArch64::X18)
1522 : .addImm(-8)
1523 : .setMIFlag(MachineInstr::FrameDestroy);
1524 : }
1525 :
1526 1372 : return true;
1527 : }
1528 :
1529 15053 : void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
1530 : BitVector &SavedRegs,
1531 : RegScavenger *RS) const {
1532 : // All calls are tail calls in GHC calling conv, and functions have no
1533 : // prologue/epilogue.
1534 30106 : if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1535 : return;
1536 :
1537 15047 : TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1538 : const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
1539 15047 : MF.getSubtarget().getRegisterInfo());
1540 15047 : AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1541 : unsigned UnspilledCSGPR = AArch64::NoRegister;
1542 : unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
1543 :
1544 15047 : MachineFrameInfo &MFI = MF.getFrameInfo();
1545 15047 : const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
1546 :
1547 15047 : unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
1548 15047 : ? RegInfo->getBaseRegister()
1549 : : (unsigned)AArch64::NoRegister;
1550 :
1551 : unsigned ExtraCSSpill = 0;
1552 : // Figure out which callee-saved registers to save/restore.
1553 316240 : for (unsigned i = 0; CSRegs[i]; ++i) {
1554 301193 : const unsigned Reg = CSRegs[i];
1555 :
1556 : // Add the base pointer register to SavedRegs if it is callee-save.
1557 301193 : if (Reg == BasePointerReg)
1558 : SavedRegs.set(Reg);
1559 :
1560 : bool RegUsed = SavedRegs.test(Reg);
1561 301193 : unsigned PairedReg = CSRegs[i ^ 1];
1562 301193 : if (!RegUsed) {
1563 474548 : if (AArch64::GPR64RegClass.contains(Reg) &&
1564 177284 : !RegInfo->isReservedReg(MF, Reg)) {
1565 : UnspilledCSGPR = Reg;
1566 : UnspilledCSGPRPaired = PairedReg;
1567 : }
1568 297264 : continue;
1569 : }
1570 :
1571 : // MachO's compact unwind format relies on all registers being stored in
1572 : // pairs.
1573 : // FIXME: the usual format is actually better if unwinding isn't needed.
1574 3929 : if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
1575 : !SavedRegs.test(PairedReg)) {
1576 : SavedRegs.set(PairedReg);
1577 769 : if (AArch64::GPR64RegClass.contains(PairedReg) &&
1578 377 : !RegInfo->isReservedReg(MF, PairedReg))
1579 : ExtraCSSpill = PairedReg;
1580 : }
1581 : }
1582 :
1583 : // Calculates the callee saved stack size.
1584 : unsigned CSStackSize = 0;
1585 15047 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
1586 15047 : const MachineRegisterInfo &MRI = MF.getRegInfo();
1587 18996 : for (unsigned Reg : SavedRegs.set_bits())
1588 3949 : CSStackSize += TRI->getRegSizeInBits(Reg, MRI) / 8;
1589 :
1590 : // Save number of saved regs, so we can easily update CSStackSize later.
1591 : unsigned NumSavedRegs = SavedRegs.count();
1592 :
1593 : // The frame record needs to be created by saving the appropriate registers
1594 15047 : unsigned EstimatedStackSize = MFI.estimateStackSize(MF);
1595 29794 : if (hasFP(MF) ||
1596 14747 : windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
1597 : SavedRegs.set(AArch64::FP);
1598 : SavedRegs.set(AArch64::LR);
1599 : }
1600 :
1601 : LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
1602 : for (unsigned Reg
1603 : : SavedRegs.set_bits()) dbgs()
1604 : << ' ' << printReg(Reg, RegInfo);
1605 : dbgs() << "\n";);
1606 :
1607 : // If any callee-saved registers are used, the frame cannot be eliminated.
1608 : bool CanEliminateFrame = SavedRegs.count() == 0;
1609 :
1610 : // The CSR spill slots have not been allocated yet, so estimateStackSize
1611 : // won't include them.
1612 15047 : unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
1613 15047 : bool BigStack = (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
1614 15047 : if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
1615 : AFI->setHasStackFrame(true);
1616 :
1617 : // Estimate if we might need to scavenge a register at some point in order
1618 : // to materialize a stack offset. If so, either spill one additional
1619 : // callee-saved register or reserve a special spill slot to facilitate
1620 : // register scavenging. If we already spilled an extra callee-saved register
1621 : // above to keep the number of spills even, we don't need to do anything else
1622 : // here.
1623 15047 : if (BigStack) {
1624 63 : if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
1625 : LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
1626 : << " to get a scratch register.\n");
1627 : SavedRegs.set(UnspilledCSGPR);
1628 : // MachO's compact unwind format relies on all registers being stored in
1629 : // pairs, so if we need to spill one extra for BigStack, then we need to
1630 : // store the pair.
1631 38 : if (produceCompactUnwindFrame(MF))
1632 : SavedRegs.set(UnspilledCSGPRPaired);
1633 : ExtraCSSpill = UnspilledCSGPRPaired;
1634 : }
1635 :
1636 : // If we didn't find an extra callee-saved register to spill, create
1637 : // an emergency spill slot.
1638 63 : if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
1639 25 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
1640 : const TargetRegisterClass &RC = AArch64::GPR64RegClass;
1641 : unsigned Size = TRI->getSpillSize(RC);
1642 : unsigned Align = TRI->getSpillAlignment(RC);
1643 25 : int FI = MFI.CreateStackObject(Size, Align, false);
1644 : RS->addScavengingFrameIndex(FI);
1645 : LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
1646 : << " as the emergency spill slot.\n");
1647 : }
1648 : }
1649 :
1650 : // Adding the size of additional 64bit GPR saves.
1651 15047 : CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
1652 30094 : unsigned AlignedCSStackSize = alignTo(CSStackSize, 16);
1653 : LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
1654 : << EstimatedStackSize + AlignedCSStackSize
1655 : << " bytes.\n");
1656 :
1657 : // Round up to register pair alignment to avoid additional SP adjustment
1658 : // instructions.
1659 : AFI->setCalleeSavedStackSize(AlignedCSStackSize);
1660 15047 : AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
1661 : }
1662 :
1663 493 : bool AArch64FrameLowering::enableStackSlotScavenging(
1664 : const MachineFunction &MF) const {
1665 : const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1666 493 : return AFI->hasCalleeSaveStackFreeSpace();
1667 : }
|
