LLVM  9.0.0svn
AArch64FrameLowering.cpp
Go to the documentation of this file.
1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of TargetFrameLowering class.
10 //
11 // On AArch64, stack frames are structured as follows:
12 //
13 // The stack grows downward.
14 //
15 // All of the individual frame areas on the frame below are optional, i.e. it's
16 // possible to create a function so that the particular area isn't present
17 // in the frame.
18 //
19 // At function entry, the "frame" looks as follows:
20 //
21 // | | Higher address
22 // |-----------------------------------|
23 // | |
24 // | arguments passed on the stack |
25 // | |
26 // |-----------------------------------| <- sp
27 // | | Lower address
28 //
29 //
30 // After the prologue has run, the frame has the following general structure.
31 // Note that this doesn't depict the case where a red-zone is used. Also,
32 // technically the last frame area (VLAs) doesn't get created until in the
33 // main function body, after the prologue is run. However, it's depicted here
34 // for completeness.
35 //
36 // | | Higher address
37 // |-----------------------------------|
38 // | |
39 // | arguments passed on the stack |
40 // | |
41 // |-----------------------------------|
42 // | |
43 // | (Win64 only) varargs from reg |
44 // | |
45 // |-----------------------------------|
46 // | |
47 // | prev_fp, prev_lr |
48 // | (a.k.a. "frame record") |
49 // |-----------------------------------| <- fp(=x29)
50 // | |
51 // | other callee-saved registers |
52 // | |
53 // |-----------------------------------|
54 // |.empty.space.to.make.part.below....|
55 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
56 // |.the.standard.16-byte.alignment....| compile time; if present)
57 // |-----------------------------------|
58 // | |
59 // | local variables of fixed size |
60 // | including spill slots |
61 // |-----------------------------------| <- bp(not defined by ABI,
62 // |.variable-sized.local.variables....| LLVM chooses X19)
63 // |.(VLAs)............................| (size of this area is unknown at
64 // |...................................| compile time)
65 // |-----------------------------------| <- sp
66 // | | Lower address
67 //
68 //
69 // To access the data in a frame, at-compile time, a constant offset must be
70 // computable from one of the pointers (fp, bp, sp) to access it. The size
71 // of the areas with a dotted background cannot be computed at compile-time
72 // if they are present, making it required to have all three of fp, bp and
73 // sp to be set up to be able to access all contents in the frame areas,
74 // assuming all of the frame areas are non-empty.
75 //
76 // For most functions, some of the frame areas are empty. For those functions,
77 // it may not be necessary to set up fp or bp:
78 // * A base pointer is definitely needed when there are both VLAs and local
79 // variables with more-than-default alignment requirements.
80 // * A frame pointer is definitely needed when there are local variables with
81 // more-than-default alignment requirements.
82 //
83 // In some cases when a base pointer is not strictly needed, it is generated
84 // anyway when offsets from the frame pointer to access local variables become
85 // so large that the offset can't be encoded in the immediate fields of loads
86 // or stores.
87 //
88 // FIXME: also explain the redzone concept.
89 // FIXME: also explain the concept of reserved call frames.
90 //
91 //===----------------------------------------------------------------------===//
92 
93 #include "AArch64FrameLowering.h"
94 #include "AArch64InstrInfo.h"
96 #include "AArch64RegisterInfo.h"
97 #include "AArch64Subtarget.h"
98 #include "AArch64TargetMachine.h"
100 #include "llvm/ADT/ScopeExit.h"
101 #include "llvm/ADT/SmallVector.h"
102 #include "llvm/ADT/Statistic.h"
118 #include "llvm/IR/Attributes.h"
119 #include "llvm/IR/CallingConv.h"
120 #include "llvm/IR/DataLayout.h"
121 #include "llvm/IR/DebugLoc.h"
122 #include "llvm/IR/Function.h"
123 #include "llvm/MC/MCAsmInfo.h"
124 #include "llvm/MC/MCDwarf.h"
126 #include "llvm/Support/Debug.h"
128 #include "llvm/Support/MathExtras.h"
132 #include <cassert>
133 #include <cstdint>
134 #include <iterator>
135 #include <vector>
136 
137 using namespace llvm;
138 
139 #define DEBUG_TYPE "frame-info"
140 
141 static cl::opt<bool> EnableRedZone("aarch64-redzone",
142  cl::desc("enable use of redzone on AArch64"),
143  cl::init(false), cl::Hidden);
144 
145 static cl::opt<bool>
146  ReverseCSRRestoreSeq("reverse-csr-restore-seq",
147  cl::desc("reverse the CSR restore sequence"),
148  cl::init(false), cl::Hidden);
149 
150 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
151 
152 /// This is the biggest offset to the stack pointer we can encode in aarch64
153 /// instructions (without using a separate calculation and a temp register).
154 /// Note that the exception here are vector stores/loads which cannot encode any
155 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
156 static const unsigned DefaultSafeSPDisplacement = 255;
157 
158 /// Look at each instruction that references stack frames and return the stack
159 /// size limit beyond which some of these instructions will require a scratch
160 /// register during their expansion later.
162  // FIXME: For now, just conservatively guestimate based on unscaled indexing
163  // range. We'll end up allocating an unnecessary spill slot a lot, but
164  // realistically that's not a big deal at this stage of the game.
165  for (MachineBasicBlock &MBB : MF) {
166  for (MachineInstr &MI : MBB) {
167  if (MI.isDebugInstr() || MI.isPseudo() ||
168  MI.getOpcode() == AArch64::ADDXri ||
169  MI.getOpcode() == AArch64::ADDSXri)
170  continue;
171 
172  for (const MachineOperand &MO : MI.operands()) {
173  if (!MO.isFI())
174  continue;
175 
176  int Offset = 0;
177  if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
179  return 0;
180  }
181  }
182  }
184 }
185 
187  if (!EnableRedZone)
188  return false;
189  // Don't use the red zone if the function explicitly asks us not to.
190  // This is typically used for kernel code.
191  if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
192  return false;
193 
194  const MachineFrameInfo &MFI = MF.getFrameInfo();
196  unsigned NumBytes = AFI->getLocalStackSize();
197 
198  return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
199 }
200 
201 /// hasFP - Return true if the specified function should have a dedicated frame
202 /// pointer register.
204  const MachineFrameInfo &MFI = MF.getFrameInfo();
205  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
206  // Win64 EH requires a frame pointer if funclets are present, as the locals
207  // are accessed off the frame pointer in both the parent function and the
208  // funclets.
209  if (MF.hasEHFunclets())
210  return true;
211  // Retain behavior of always omitting the FP for leaf functions when possible.
212  if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
213  return true;
214  if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
215  MFI.hasStackMap() || MFI.hasPatchPoint() ||
216  RegInfo->needsStackRealignment(MF))
217  return true;
218  // With large callframes around we may need to use FP to access the scavenging
219  // emergency spillslot.
220  //
221  // Unfortunately some calls to hasFP() like machine verifier ->
222  // getReservedReg() -> hasFP in the middle of global isel are too early
223  // to know the max call frame size. Hopefully conservatively returning "true"
224  // in those cases is fine.
225  // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
226  if (!MFI.isMaxCallFrameSizeComputed() ||
228  return true;
229 
230  return false;
231 }
232 
233 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
234 /// not required, we reserve argument space for call sites in the function
235 /// immediately on entry to the current function. This eliminates the need for
236 /// add/sub sp brackets around call sites. Returns true if the call frame is
237 /// included as part of the stack frame.
238 bool
240  return !MF.getFrameInfo().hasVarSizedObjects();
241 }
242 
246  const AArch64InstrInfo *TII =
247  static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
248  DebugLoc DL = I->getDebugLoc();
249  unsigned Opc = I->getOpcode();
250  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
251  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
252 
253  if (!hasReservedCallFrame(MF)) {
254  unsigned Align = getStackAlignment();
255 
256  int64_t Amount = I->getOperand(0).getImm();
257  Amount = alignTo(Amount, Align);
258  if (!IsDestroy)
259  Amount = -Amount;
260 
261  // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
262  // doesn't have to pop anything), then the first operand will be zero too so
263  // this adjustment is a no-op.
264  if (CalleePopAmount == 0) {
265  // FIXME: in-function stack adjustment for calls is limited to 24-bits
266  // because there's no guaranteed temporary register available.
267  //
268  // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
269  // 1) For offset <= 12-bit, we use LSL #0
270  // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
271  // LSL #0, and the other uses LSL #12.
272  //
273  // Most call frames will be allocated at the start of a function so
274  // this is OK, but it is a limitation that needs dealing with.
275  assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
276  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
277  }
278  } else if (CalleePopAmount != 0) {
279  // If the calling convention demands that the callee pops arguments from the
280  // stack, we want to add it back if we have a reserved call frame.
281  assert(CalleePopAmount < 0xffffff && "call frame too large");
282  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
283  TII);
284  }
285  return MBB.erase(I);
286 }
287 
289  // The function should be signed in the following situations:
290  // - sign-return-address=all
291  // - sign-return-address=non-leaf and the functions spills the LR
292 
293  const Function &F = MF.getFunction();
294  if (!F.hasFnAttribute("sign-return-address"))
295  return false;
296 
297  StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
298  if (Scope.equals("none"))
299  return false;
300 
301  if (Scope.equals("all"))
302  return true;
303 
304  assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
305 
306  for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
307  if (Info.getReg() == AArch64::LR)
308  return true;
309 
310  return false;
311 }
312 
315  MachineFunction &MF = *MBB.getParent();
316  MachineFrameInfo &MFI = MF.getFrameInfo();
317  const TargetSubtargetInfo &STI = MF.getSubtarget();
318  const MCRegisterInfo *MRI = STI.getRegisterInfo();
319  const TargetInstrInfo *TII = STI.getInstrInfo();
320  DebugLoc DL = MBB.findDebugLoc(MBBI);
321 
322  // Add callee saved registers to move list.
323  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
324  if (CSI.empty())
325  return;
326 
327  for (const auto &Info : CSI) {
328  unsigned Reg = Info.getReg();
329  int64_t Offset =
330  MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
331  unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
332  unsigned CFIIndex = MF.addFrameInst(
333  MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
334  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
335  .addCFIIndex(CFIIndex)
337  }
338 }
339 
340 // Find a scratch register that we can use at the start of the prologue to
341 // re-align the stack pointer. We avoid using callee-save registers since they
342 // may appear to be free when this is called from canUseAsPrologue (during
343 // shrink wrapping), but then no longer be free when this is called from
344 // emitPrologue.
345 //
346 // FIXME: This is a bit conservative, since in the above case we could use one
347 // of the callee-save registers as a scratch temp to re-align the stack pointer,
348 // but we would then have to make sure that we were in fact saving at least one
349 // callee-save register in the prologue, which is additional complexity that
350 // doesn't seem worth the benefit.
352  MachineFunction *MF = MBB->getParent();
353 
354  // If MBB is an entry block, use X9 as the scratch register
355  if (&MF->front() == MBB)
356  return AArch64::X9;
357 
358  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
359  const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
360  LivePhysRegs LiveRegs(TRI);
361  LiveRegs.addLiveIns(*MBB);
362 
363  // Mark callee saved registers as used so we will not choose them.
364  const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
365  for (unsigned i = 0; CSRegs[i]; ++i)
366  LiveRegs.addReg(CSRegs[i]);
367 
368  // Prefer X9 since it was historically used for the prologue scratch reg.
369  const MachineRegisterInfo &MRI = MF->getRegInfo();
370  if (LiveRegs.available(MRI, AArch64::X9))
371  return AArch64::X9;
372 
373  for (unsigned Reg : AArch64::GPR64RegClass) {
374  if (LiveRegs.available(MRI, Reg))
375  return Reg;
376  }
377  return AArch64::NoRegister;
378 }
379 
381  const MachineBasicBlock &MBB) const {
382  const MachineFunction *MF = MBB.getParent();
383  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
384  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
385  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
386 
387  // Don't need a scratch register if we're not going to re-align the stack.
388  if (!RegInfo->needsStackRealignment(*MF))
389  return true;
390  // Otherwise, we can use any block as long as it has a scratch register
391  // available.
392  return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
393 }
394 
396  unsigned StackSizeInBytes) {
397  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
398  if (!Subtarget.isTargetWindows())
399  return false;
400  const Function &F = MF.getFunction();
401  // TODO: When implementing stack protectors, take that into account
402  // for the probe threshold.
403  unsigned StackProbeSize = 4096;
404  if (F.hasFnAttribute("stack-probe-size"))
405  F.getFnAttribute("stack-probe-size")
407  .getAsInteger(0, StackProbeSize);
408  return (StackSizeInBytes >= StackProbeSize) &&
409  !F.hasFnAttribute("no-stack-arg-probe");
410 }
411 
412 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
413  MachineFunction &MF, unsigned StackBumpBytes) const {
415  const MachineFrameInfo &MFI = MF.getFrameInfo();
416  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
417  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
418 
419  if (AFI->getLocalStackSize() == 0)
420  return false;
421 
422  // 512 is the maximum immediate for stp/ldp that will be used for
423  // callee-save save/restores
424  if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
425  return false;
426 
427  if (MFI.hasVarSizedObjects())
428  return false;
429 
430  if (RegInfo->needsStackRealignment(MF))
431  return false;
432 
433  // This isn't strictly necessary, but it simplifies things a bit since the
434  // current RedZone handling code assumes the SP is adjusted by the
435  // callee-save save/restore code.
436  if (canUseRedZone(MF))
437  return false;
438 
439  return true;
440 }
441 
442 // Given a load or a store instruction, generate an appropriate unwinding SEH
443 // code on Windows.
445  const TargetInstrInfo &TII,
447  unsigned Opc = MBBI->getOpcode();
448  MachineBasicBlock *MBB = MBBI->getParent();
449  MachineFunction &MF = *MBB->getParent();
450  DebugLoc DL = MBBI->getDebugLoc();
451  unsigned ImmIdx = MBBI->getNumOperands() - 1;
452  int Imm = MBBI->getOperand(ImmIdx).getImm();
454  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
455  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
456 
457  switch (Opc) {
458  default:
459  llvm_unreachable("No SEH Opcode for this instruction");
460  case AArch64::LDPDpost:
461  Imm = -Imm;
463  case AArch64::STPDpre: {
464  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
465  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
466  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
467  .addImm(Reg0)
468  .addImm(Reg1)
469  .addImm(Imm * 8)
470  .setMIFlag(Flag);
471  break;
472  }
473  case AArch64::LDPXpost:
474  Imm = -Imm;
476  case AArch64::STPXpre: {
477  unsigned Reg0 = MBBI->getOperand(1).getReg();
478  unsigned Reg1 = MBBI->getOperand(2).getReg();
479  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
480  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
481  .addImm(Imm * 8)
482  .setMIFlag(Flag);
483  else
484  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
485  .addImm(RegInfo->getSEHRegNum(Reg0))
486  .addImm(RegInfo->getSEHRegNum(Reg1))
487  .addImm(Imm * 8)
488  .setMIFlag(Flag);
489  break;
490  }
491  case AArch64::LDRDpost:
492  Imm = -Imm;
494  case AArch64::STRDpre: {
495  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
496  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
497  .addImm(Reg)
498  .addImm(Imm)
499  .setMIFlag(Flag);
500  break;
501  }
502  case AArch64::LDRXpost:
503  Imm = -Imm;
505  case AArch64::STRXpre: {
506  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
507  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
508  .addImm(Reg)
509  .addImm(Imm)
510  .setMIFlag(Flag);
511  break;
512  }
513  case AArch64::STPDi:
514  case AArch64::LDPDi: {
515  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
516  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
517  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
518  .addImm(Reg0)
519  .addImm(Reg1)
520  .addImm(Imm * 8)
521  .setMIFlag(Flag);
522  break;
523  }
524  case AArch64::STPXi:
525  case AArch64::LDPXi: {
526  unsigned Reg0 = MBBI->getOperand(0).getReg();
527  unsigned Reg1 = MBBI->getOperand(1).getReg();
528  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
529  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
530  .addImm(Imm * 8)
531  .setMIFlag(Flag);
532  else
533  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
534  .addImm(RegInfo->getSEHRegNum(Reg0))
535  .addImm(RegInfo->getSEHRegNum(Reg1))
536  .addImm(Imm * 8)
537  .setMIFlag(Flag);
538  break;
539  }
540  case AArch64::STRXui:
541  case AArch64::LDRXui: {
542  int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
543  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
544  .addImm(Reg)
545  .addImm(Imm * 8)
546  .setMIFlag(Flag);
547  break;
548  }
549  case AArch64::STRDui:
550  case AArch64::LDRDui: {
551  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
552  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
553  .addImm(Reg)
554  .addImm(Imm * 8)
555  .setMIFlag(Flag);
556  break;
557  }
558  }
559  auto I = MBB->insertAfter(MBBI, MIB);
560  return I;
561 }
562 
563 // Fix up the SEH opcode associated with the save/restore instruction.
565  unsigned LocalStackSize) {
566  MachineOperand *ImmOpnd = nullptr;
567  unsigned ImmIdx = MBBI->getNumOperands() - 1;
568  switch (MBBI->getOpcode()) {
569  default:
570  llvm_unreachable("Fix the offset in the SEH instruction");
571  case AArch64::SEH_SaveFPLR:
572  case AArch64::SEH_SaveRegP:
573  case AArch64::SEH_SaveReg:
574  case AArch64::SEH_SaveFRegP:
575  case AArch64::SEH_SaveFReg:
576  ImmOpnd = &MBBI->getOperand(ImmIdx);
577  break;
578  }
579  if (ImmOpnd)
580  ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
581 }
582 
583 // Convert callee-save register save/restore instruction to do stack pointer
584 // decrement/increment to allocate/deallocate the callee-save stack area by
585 // converting store/load to use pre/post increment version.
588  const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
589  bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) {
590  // Ignore instructions that do not operate on SP, i.e. shadow call stack
591  // instructions and associated CFI instruction.
592  while (MBBI->getOpcode() == AArch64::STRXpost ||
593  MBBI->getOpcode() == AArch64::LDRXpre ||
594  MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
595  if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
596  assert(MBBI->getOperand(0).getReg() != AArch64::SP);
597  ++MBBI;
598  }
599  unsigned NewOpc;
600  int Scale = 1;
601  switch (MBBI->getOpcode()) {
602  default:
603  llvm_unreachable("Unexpected callee-save save/restore opcode!");
604  case AArch64::STPXi:
605  NewOpc = AArch64::STPXpre;
606  Scale = 8;
607  break;
608  case AArch64::STPDi:
609  NewOpc = AArch64::STPDpre;
610  Scale = 8;
611  break;
612  case AArch64::STPQi:
613  NewOpc = AArch64::STPQpre;
614  Scale = 16;
615  break;
616  case AArch64::STRXui:
617  NewOpc = AArch64::STRXpre;
618  break;
619  case AArch64::STRDui:
620  NewOpc = AArch64::STRDpre;
621  break;
622  case AArch64::STRQui:
623  NewOpc = AArch64::STRQpre;
624  break;
625  case AArch64::LDPXi:
626  NewOpc = AArch64::LDPXpost;
627  Scale = 8;
628  break;
629  case AArch64::LDPDi:
630  NewOpc = AArch64::LDPDpost;
631  Scale = 8;
632  break;
633  case AArch64::LDPQi:
634  NewOpc = AArch64::LDPQpost;
635  Scale = 16;
636  break;
637  case AArch64::LDRXui:
638  NewOpc = AArch64::LDRXpost;
639  break;
640  case AArch64::LDRDui:
641  NewOpc = AArch64::LDRDpost;
642  break;
643  case AArch64::LDRQui:
644  NewOpc = AArch64::LDRQpost;
645  break;
646  }
647  // Get rid of the SEH code associated with the old instruction.
648  if (NeedsWinCFI) {
649  auto SEH = std::next(MBBI);
651  SEH->eraseFromParent();
652  }
653 
654  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
655  MIB.addReg(AArch64::SP, RegState::Define);
656 
657  // Copy all operands other than the immediate offset.
658  unsigned OpndIdx = 0;
659  for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
660  ++OpndIdx)
661  MIB.add(MBBI->getOperand(OpndIdx));
662 
663  assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
664  "Unexpected immediate offset in first/last callee-save save/restore "
665  "instruction!");
666  assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
667  "Unexpected base register in callee-save save/restore instruction!");
668  assert(CSStackSizeInc % Scale == 0);
669  MIB.addImm(CSStackSizeInc / Scale);
670 
671  MIB.setMIFlags(MBBI->getFlags());
672  MIB.setMemRefs(MBBI->memoperands());
673 
674  // Generate a new SEH code that corresponds to the new instruction.
675  if (NeedsWinCFI) {
676  *HasWinCFI = true;
677  InsertSEH(*MIB, *TII,
679  }
680 
681  return std::prev(MBB.erase(MBBI));
682 }
683 
684 // Fixup callee-save register save/restore instructions to take into account
685 // combined SP bump by adding the local stack size to the stack offsets.
687  unsigned LocalStackSize,
688  bool NeedsWinCFI,
689  bool *HasWinCFI) {
691  return;
692 
693  unsigned Opc = MI.getOpcode();
694 
695  // Ignore instructions that do not operate on SP, i.e. shadow call stack
696  // instructions and associated CFI instruction.
697  if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
698  Opc == AArch64::CFI_INSTRUCTION) {
699  if (Opc != AArch64::CFI_INSTRUCTION)
700  assert(MI.getOperand(0).getReg() != AArch64::SP);
701  return;
702  }
703 
704  unsigned Scale;
705  switch (Opc) {
706  case AArch64::STPXi:
707  case AArch64::STRXui:
708  case AArch64::STPDi:
709  case AArch64::STRDui:
710  case AArch64::LDPXi:
711  case AArch64::LDRXui:
712  case AArch64::LDPDi:
713  case AArch64::LDRDui:
714  Scale = 8;
715  break;
716  case AArch64::STPQi:
717  case AArch64::STRQui:
718  case AArch64::LDPQi:
719  case AArch64::LDRQui:
720  Scale = 16;
721  break;
722  default:
723  llvm_unreachable("Unexpected callee-save save/restore opcode!");
724  }
725 
726  unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
727  assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
728  "Unexpected base register in callee-save save/restore instruction!");
729  // Last operand is immediate offset that needs fixing.
730  MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
731  // All generated opcodes have scaled offsets.
732  assert(LocalStackSize % Scale == 0);
733  OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
734 
735  if (NeedsWinCFI) {
736  *HasWinCFI = true;
737  auto MBBI = std::next(MachineBasicBlock::iterator(MI));
738  assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
740  "Expecting a SEH instruction");
741  fixupSEHOpcode(MBBI, LocalStackSize);
742  }
743 }
744 
746  MachineBasicBlock::iterator FirstSPPopI,
747  MachineBasicBlock::iterator LastPopI) {
748  // Sometimes (when we restore in the same order as we save), we can end up
749  // with code like this:
750  //
751  // ldp x26, x25, [sp]
752  // ldp x24, x23, [sp, #16]
753  // ldp x22, x21, [sp, #32]
754  // ldp x20, x19, [sp, #48]
755  // add sp, sp, #64
756  //
757  // In this case, it is always better to put the first ldp at the end, so
758  // that the load-store optimizer can run and merge the ldp and the add into
759  // a post-index ldp.
760  // If we managed to grab the first pop instruction, move it to the end.
762  MBB.splice(FirstSPPopI, &MBB, LastPopI);
763  // We should end up with something like this now:
764  //
765  // ldp x24, x23, [sp, #16]
766  // ldp x22, x21, [sp, #32]
767  // ldp x20, x19, [sp, #48]
768  // ldp x26, x25, [sp]
769  // add sp, sp, #64
770  //
771  // and the load-store optimizer can merge the last two instructions into:
772  //
773  // ldp x26, x25, [sp], #64
774  //
775 }
776 
778  const Function &F = MF.getFunction();
779  if (!F.hasFnAttribute("sign-return-address-key"))
780  return true;
781 
782  const StringRef Key =
783  F.getFnAttribute("sign-return-address-key").getValueAsString();
784  assert(Key.equals_lower("a_key") || Key.equals_lower("b_key"));
785  return Key.equals_lower("a_key");
786 }
787 
788 static bool needsWinCFI(const MachineFunction &MF) {
789  const Function &F = MF.getFunction();
790  return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
792 }
793 
795  MachineBasicBlock &MBB) const {
796  MachineBasicBlock::iterator MBBI = MBB.begin();
797  const MachineFrameInfo &MFI = MF.getFrameInfo();
798  const Function &F = MF.getFunction();
799  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
800  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
801  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
802  MachineModuleInfo &MMI = MF.getMMI();
804  bool needsFrameMoves = (MMI.hasDebugInfo() || F.needsUnwindTableEntry()) &&
806  bool HasFP = hasFP(MF);
807  bool NeedsWinCFI = needsWinCFI(MF);
808  bool HasWinCFI = false;
809  auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
810 
811  bool IsFunclet = MBB.isEHFuncletEntry();
812 
813  // At this point, we're going to decide whether or not the function uses a
814  // redzone. In most cases, the function doesn't have a redzone so let's
815  // assume that's false and set it to true in the case that there's a redzone.
816  AFI->setHasRedZone(false);
817 
818  // Debug location must be unknown since the first debug location is used
819  // to determine the end of the prologue.
820  DebugLoc DL;
821 
822  if (ShouldSignReturnAddress(MF)) {
823  if (ShouldSignWithAKey(MF))
824  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
825  .setMIFlag(MachineInstr::FrameSetup);
826  else {
827  BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
828  .setMIFlag(MachineInstr::FrameSetup);
829  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
830  .setMIFlag(MachineInstr::FrameSetup);
831  }
832 
833  unsigned CFIIndex =
835  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
836  .addCFIIndex(CFIIndex)
838  }
839 
840  // All calls are tail calls in GHC calling conv, and functions have no
841  // prologue/epilogue.
843  return;
844 
845  // getStackSize() includes all the locals in its size calculation. We don't
846  // include these locals when computing the stack size of a funclet, as they
847  // are allocated in the parent's stack frame and accessed via the frame
848  // pointer from the funclet. We only save the callee saved registers in the
849  // funclet, which are really the callee saved registers of the parent
850  // function, including the funclet.
851  int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
852  : (int)MFI.getStackSize();
853  if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
854  assert(!HasFP && "unexpected function without stack frame but with FP");
855  // All of the stack allocation is for locals.
856  AFI->setLocalStackSize(NumBytes);
857  if (!NumBytes)
858  return;
859  // REDZONE: If the stack size is less than 128 bytes, we don't need
860  // to actually allocate.
861  if (canUseRedZone(MF)) {
862  AFI->setHasRedZone(true);
863  ++NumRedZoneFunctions;
864  } else {
865  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
866  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
867  if (!NeedsWinCFI) {
868  // Label used to tie together the PROLOG_LABEL and the MachineMoves.
869  MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
870  // Encode the stack size of the leaf function.
871  unsigned CFIIndex = MF.addFrameInst(
872  MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
873  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
874  .addCFIIndex(CFIIndex)
876  }
877  }
878 
879  if (NeedsWinCFI) {
880  HasWinCFI = true;
881  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
882  .setMIFlag(MachineInstr::FrameSetup);
883  }
884 
885  return;
886  }
887 
888  bool IsWin64 =
890  // Var args are accounted for in the containing function, so don't
891  // include them for funclets.
892  unsigned FixedObject = (IsWin64 && !IsFunclet) ?
893  alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
894 
895  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
896  // All of the remaining stack allocations are for locals.
897  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
898  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
899  if (CombineSPBump) {
900  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
901  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
902  NumBytes = 0;
903  } else if (PrologueSaveSize != 0) {
905  MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI);
906  NumBytes -= PrologueSaveSize;
907  }
908  assert(NumBytes >= 0 && "Negative stack allocation size!?");
909 
910  // Move past the saves of the callee-saved registers, fixing up the offsets
911  // and pre-inc if we decided to combine the callee-save and local stack
912  // pointer bump above.
913  MachineBasicBlock::iterator End = MBB.end();
914  while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
915  if (CombineSPBump)
917  NeedsWinCFI, &HasWinCFI);
918  ++MBBI;
919  }
920 
921  // The code below is not applicable to funclets. We have emitted all the SEH
922  // opcodes that we needed to emit. The FP and BP belong to the containing
923  // function.
924  if (IsFunclet) {
925  if (NeedsWinCFI) {
926  HasWinCFI = true;
927  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
928  .setMIFlag(MachineInstr::FrameSetup);
929  }
930 
931  // SEH funclets are passed the frame pointer in X1. If the parent
932  // function uses the base register, then the base register is used
933  // directly, and is not retrieved from X1.
934  if (F.hasPersonalityFn()) {
936  if (isAsynchronousEHPersonality(Per)) {
937  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
938  .addReg(AArch64::X1).setMIFlag(MachineInstr::FrameSetup);
939  MBB.addLiveIn(AArch64::X1);
940  }
941  }
942 
943  return;
944  }
945 
946  if (HasFP) {
947  // Only set up FP if we actually need to. Frame pointer is fp =
948  // sp - fixedobject - 16.
949  int FPOffset = AFI->getCalleeSavedStackSize() - 16;
950  if (CombineSPBump)
951  FPOffset += AFI->getLocalStackSize();
952 
953  // Issue sub fp, sp, FPOffset or
954  // mov fp,sp when FPOffset is zero.
955  // Note: All stores of callee-saved registers are marked as "FrameSetup".
956  // This code marks the instruction(s) that set the FP also.
957  emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
958  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
959  }
960 
961  if (windowsRequiresStackProbe(MF, NumBytes)) {
962  uint32_t NumWords = NumBytes >> 4;
963  if (NeedsWinCFI) {
964  HasWinCFI = true;
965  // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
966  // exceed this amount. We need to move at most 2^24 - 1 into x15.
967  // This is at most two instructions, MOVZ follwed by MOVK.
968  // TODO: Fix to use multiple stack alloc unwind codes for stacks
969  // exceeding 256MB in size.
970  if (NumBytes >= (1 << 28))
971  report_fatal_error("Stack size cannot exceed 256MB for stack "
972  "unwinding purposes");
973 
974  uint32_t LowNumWords = NumWords & 0xFFFF;
975  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
976  .addImm(LowNumWords)
979  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
980  .setMIFlag(MachineInstr::FrameSetup);
981  if ((NumWords & 0xFFFF0000) != 0) {
982  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
983  .addReg(AArch64::X15)
984  .addImm((NumWords & 0xFFFF0000) >> 16) // High half
987  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
988  .setMIFlag(MachineInstr::FrameSetup);
989  }
990  } else {
991  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
992  .addImm(NumWords)
994  }
995 
996  switch (MF.getTarget().getCodeModel()) {
997  case CodeModel::Tiny:
998  case CodeModel::Small:
999  case CodeModel::Medium:
1000  case CodeModel::Kernel:
1001  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1002  .addExternalSymbol("__chkstk")
1003  .addReg(AArch64::X15, RegState::Implicit)
1008  if (NeedsWinCFI) {
1009  HasWinCFI = true;
1010  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1011  .setMIFlag(MachineInstr::FrameSetup);
1012  }
1013  break;
1014  case CodeModel::Large:
1015  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1016  .addReg(AArch64::X16, RegState::Define)
1017  .addExternalSymbol("__chkstk")
1018  .addExternalSymbol("__chkstk")
1020  if (NeedsWinCFI) {
1021  HasWinCFI = true;
1022  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1023  .setMIFlag(MachineInstr::FrameSetup);
1024  }
1025 
1026  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
1027  .addReg(AArch64::X16, RegState::Kill)
1028  .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1033  if (NeedsWinCFI) {
1034  HasWinCFI = true;
1035  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1036  .setMIFlag(MachineInstr::FrameSetup);
1037  }
1038  break;
1039  }
1040 
1041  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1042  .addReg(AArch64::SP, RegState::Kill)
1043  .addReg(AArch64::X15, RegState::Kill)
1046  if (NeedsWinCFI) {
1047  HasWinCFI = true;
1048  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1049  .addImm(NumBytes)
1051  }
1052  NumBytes = 0;
1053  }
1054 
1055  // Allocate space for the rest of the frame.
1056  if (NumBytes) {
1057  const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
1058  unsigned scratchSPReg = AArch64::SP;
1059 
1060  if (NeedsRealignment) {
1061  scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1062  assert(scratchSPReg != AArch64::NoRegister);
1063  }
1064 
1065  // If we're a leaf function, try using the red zone.
1066  if (!canUseRedZone(MF))
1067  // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1068  // the correct value here, as NumBytes also includes padding bytes,
1069  // which shouldn't be counted here.
1070  emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
1071  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1072 
1073  if (NeedsRealignment) {
1074  const unsigned Alignment = MFI.getMaxAlignment();
1075  const unsigned NrBitsToZero = countTrailingZeros(Alignment);
1076  assert(NrBitsToZero > 1);
1077  assert(scratchSPReg != AArch64::SP);
1078 
1079  // SUB X9, SP, NumBytes
1080  // -- X9 is temporary register, so shouldn't contain any live data here,
1081  // -- free to use. This is already produced by emitFrameOffset above.
1082  // AND SP, X9, 0b11111...0000
1083  // The logical immediates have a non-trivial encoding. The following
1084  // formula computes the encoded immediate with all ones but
1085  // NrBitsToZero zero bits as least significant bits.
1086  uint32_t andMaskEncoded = (1 << 12) // = N
1087  | ((64 - NrBitsToZero) << 6) // immr
1088  | ((64 - NrBitsToZero - 1) << 0); // imms
1089 
1090  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1091  .addReg(scratchSPReg, RegState::Kill)
1092  .addImm(andMaskEncoded);
1093  AFI->setStackRealigned(true);
1094  if (NeedsWinCFI) {
1095  HasWinCFI = true;
1096  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1097  .addImm(NumBytes & andMaskEncoded)
1099  }
1100  }
1101  }
1102 
1103  // If we need a base pointer, set it up here. It's whatever the value of the
1104  // stack pointer is at this point. Any variable size objects will be allocated
1105  // after this, so we can still use the base pointer to reference locals.
1106  //
1107  // FIXME: Clarify FrameSetup flags here.
1108  // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1109  // needed.
1110  if (RegInfo->hasBasePointer(MF)) {
1111  TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1112  false);
1113  if (NeedsWinCFI) {
1114  HasWinCFI = true;
1115  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1116  .setMIFlag(MachineInstr::FrameSetup);
1117  }
1118  }
1119 
1120  // The very last FrameSetup instruction indicates the end of prologue. Emit a
1121  // SEH opcode indicating the prologue end.
1122  if (NeedsWinCFI && HasWinCFI) {
1123  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1124  .setMIFlag(MachineInstr::FrameSetup);
1125  }
1126 
1127  if (needsFrameMoves) {
1128  const DataLayout &TD = MF.getDataLayout();
1129  const int StackGrowth = -TD.getPointerSize(0);
1130  unsigned FramePtr = RegInfo->getFrameRegister(MF);
1131  // An example of the prologue:
1132  //
1133  // .globl __foo
1134  // .align 2
1135  // __foo:
1136  // Ltmp0:
1137  // .cfi_startproc
1138  // .cfi_personality 155, ___gxx_personality_v0
1139  // Leh_func_begin:
1140  // .cfi_lsda 16, Lexception33
1141  //
1142  // stp xa,bx, [sp, -#offset]!
1143  // ...
1144  // stp x28, x27, [sp, #offset-32]
1145  // stp fp, lr, [sp, #offset-16]
1146  // add fp, sp, #offset - 16
1147  // sub sp, sp, #1360
1148  //
1149  // The Stack:
1150  // +-------------------------------------------+
1151  // 10000 | ........ | ........ | ........ | ........ |
1152  // 10004 | ........ | ........ | ........ | ........ |
1153  // +-------------------------------------------+
1154  // 10008 | ........ | ........ | ........ | ........ |
1155  // 1000c | ........ | ........ | ........ | ........ |
1156  // +===========================================+
1157  // 10010 | X28 Register |
1158  // 10014 | X28 Register |
1159  // +-------------------------------------------+
1160  // 10018 | X27 Register |
1161  // 1001c | X27 Register |
1162  // +===========================================+
1163  // 10020 | Frame Pointer |
1164  // 10024 | Frame Pointer |
1165  // +-------------------------------------------+
1166  // 10028 | Link Register |
1167  // 1002c | Link Register |
1168  // +===========================================+
1169  // 10030 | ........ | ........ | ........ | ........ |
1170  // 10034 | ........ | ........ | ........ | ........ |
1171  // +-------------------------------------------+
1172  // 10038 | ........ | ........ | ........ | ........ |
1173  // 1003c | ........ | ........ | ........ | ........ |
1174  // +-------------------------------------------+
1175  //
1176  // [sp] = 10030 :: >>initial value<<
1177  // sp = 10020 :: stp fp, lr, [sp, #-16]!
1178  // fp = sp == 10020 :: mov fp, sp
1179  // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1180  // sp == 10010 :: >>final value<<
1181  //
1182  // The frame pointer (w29) points to address 10020. If we use an offset of
1183  // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1184  // for w27, and -32 for w28:
1185  //
1186  // Ltmp1:
1187  // .cfi_def_cfa w29, 16
1188  // Ltmp2:
1189  // .cfi_offset w30, -8
1190  // Ltmp3:
1191  // .cfi_offset w29, -16
1192  // Ltmp4:
1193  // .cfi_offset w27, -24
1194  // Ltmp5:
1195  // .cfi_offset w28, -32
1196 
1197  if (HasFP) {
1198  // Define the current CFA rule to use the provided FP.
1199  unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1200  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
1201  nullptr, Reg, 2 * StackGrowth - FixedObject));
1202  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1203  .addCFIIndex(CFIIndex)
1205  } else {
1206  // Encode the stack size of the leaf function.
1207  unsigned CFIIndex = MF.addFrameInst(
1209  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1210  .addCFIIndex(CFIIndex)
1212  }
1213 
1214  // Now emit the moves for whatever callee saved regs we have (including FP,
1215  // LR if those are saved).
1216  emitCalleeSavedFrameMoves(MBB, MBBI);
1217  }
1218 }
1219 
1221  MachineBasicBlock &MBB) {
1222  if (!ShouldSignReturnAddress(MF))
1223  return;
1224  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1225  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1226 
1228  DebugLoc DL;
1229  if (MBBI != MBB.end())
1230  DL = MBBI->getDebugLoc();
1231 
1232  // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1233  // this instruction can safely used for any v8a architecture.
1234  // From v8.3a onwards there are optimised authenticate LR and return
1235  // instructions, namely RETA{A,B}, that can be used instead.
1236  if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
1237  MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1238  BuildMI(MBB, MBBI, DL,
1239  TII->get(ShouldSignWithAKey(MF) ? AArch64::RETAA : AArch64::RETAB))
1240  .copyImplicitOps(*MBBI);
1241  MBB.erase(MBBI);
1242  } else {
1243  BuildMI(
1244  MBB, MBBI, DL,
1245  TII->get(ShouldSignWithAKey(MF) ? AArch64::AUTIASP : AArch64::AUTIBSP))
1246  .setMIFlag(MachineInstr::FrameDestroy);
1247  }
1248 }
1249 
1250 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1251  switch (MI.getOpcode()) {
1252  default:
1253  return false;
1254  case AArch64::CATCHRET:
1255  case AArch64::CLEANUPRET:
1256  return true;
1257  }
1258 }
1259 
1261  MachineBasicBlock &MBB) const {
1263  MachineFrameInfo &MFI = MF.getFrameInfo();
1264  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1265  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1266  DebugLoc DL;
1267  bool IsTailCallReturn = false;
1268  bool NeedsWinCFI = needsWinCFI(MF);
1269  bool HasWinCFI = false;
1270  bool IsFunclet = false;
1271  auto WinCFI = make_scope_exit([&]() {
1272  if (!MF.hasWinCFI())
1273  MF.setHasWinCFI(HasWinCFI);
1274  });
1275 
1276  if (MBB.end() != MBBI) {
1277  DL = MBBI->getDebugLoc();
1278  unsigned RetOpcode = MBBI->getOpcode();
1279  IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
1280  RetOpcode == AArch64::TCRETURNri ||
1281  RetOpcode == AArch64::TCRETURNriBTI;
1282  IsFunclet = isFuncletReturnInstr(*MBBI);
1283  }
1284 
1285  int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
1286  : MFI.getStackSize();
1288 
1289  // All calls are tail calls in GHC calling conv, and functions have no
1290  // prologue/epilogue.
1292  return;
1293 
1294  // Initial and residual are named for consistency with the prologue. Note that
1295  // in the epilogue, the residual adjustment is executed first.
1296  uint64_t ArgumentPopSize = 0;
1297  if (IsTailCallReturn) {
1298  MachineOperand &StackAdjust = MBBI->getOperand(1);
1299 
1300  // For a tail-call in a callee-pops-arguments environment, some or all of
1301  // the stack may actually be in use for the call's arguments, this is
1302  // calculated during LowerCall and consumed here...
1303  ArgumentPopSize = StackAdjust.getImm();
1304  } else {
1305  // ... otherwise the amount to pop is *all* of the argument space,
1306  // conveniently stored in the MachineFunctionInfo by
1307  // LowerFormalArguments. This will, of course, be zero for the C calling
1308  // convention.
1309  ArgumentPopSize = AFI->getArgumentStackToRestore();
1310  }
1311 
1312  // The stack frame should be like below,
1313  //
1314  // ---------------------- ---
1315  // | | |
1316  // | BytesInStackArgArea| CalleeArgStackSize
1317  // | (NumReusableBytes) | (of tail call)
1318  // | | ---
1319  // | | |
1320  // ---------------------| --- |
1321  // | | | |
1322  // | CalleeSavedReg | | |
1323  // | (CalleeSavedStackSize)| | |
1324  // | | | |
1325  // ---------------------| | NumBytes
1326  // | | StackSize (StackAdjustUp)
1327  // | LocalStackSize | | |
1328  // | (covering callee | | |
1329  // | args) | | |
1330  // | | | |
1331  // ---------------------- --- ---
1332  //
1333  // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1334  // = StackSize + ArgumentPopSize
1335  //
1336  // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1337  // it as the 2nd argument of AArch64ISD::TC_RETURN.
1338 
1339  auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1340 
1341  bool IsWin64 =
1342  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1343  // Var args are accounted for in the containing function, so don't
1344  // include them for funclets.
1345  unsigned FixedObject =
1346  (IsWin64 && !IsFunclet) ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1347 
1348  uint64_t AfterCSRPopSize = ArgumentPopSize;
1349  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1350  // We cannot rely on the local stack size set in emitPrologue if the function
1351  // has funclets, as funclets have different local stack size requirements, and
1352  // the current value set in emitPrologue may be that of the containing
1353  // function.
1354  if (MF.hasEHFunclets())
1355  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1356  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1357  // Assume we can't combine the last pop with the sp restore.
1358 
1359  if (!CombineSPBump && PrologueSaveSize != 0) {
1360  MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
1362  Pop = std::prev(Pop);
1363  // Converting the last ldp to a post-index ldp is valid only if the last
1364  // ldp's offset is 0.
1365  const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1366  // If the offset is 0, convert it to a post-index ldp.
1367  if (OffsetOp.getImm() == 0)
1369  MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false);
1370  else {
1371  // If not, make sure to emit an add after the last ldp.
1372  // We're doing this by transfering the size to be restored from the
1373  // adjustment *before* the CSR pops to the adjustment *after* the CSR
1374  // pops.
1375  AfterCSRPopSize += PrologueSaveSize;
1376  }
1377  }
1378 
1379  // Move past the restores of the callee-saved registers.
1380  // If we plan on combining the sp bump of the local stack size and the callee
1381  // save stack size, we might need to adjust the CSR save and restore offsets.
1383  MachineBasicBlock::iterator Begin = MBB.begin();
1384  while (LastPopI != Begin) {
1385  --LastPopI;
1386  if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
1387  ++LastPopI;
1388  break;
1389  } else if (CombineSPBump)
1391  NeedsWinCFI, &HasWinCFI);
1392  }
1393 
1394  if (NeedsWinCFI) {
1395  HasWinCFI = true;
1396  BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1397  .setMIFlag(MachineInstr::FrameDestroy);
1398  }
1399 
1400  // If there is a single SP update, insert it before the ret and we're done.
1401  if (CombineSPBump) {
1402  emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1403  NumBytes + AfterCSRPopSize, TII, MachineInstr::FrameDestroy,
1404  false, NeedsWinCFI, &HasWinCFI);
1405  if (NeedsWinCFI && HasWinCFI)
1406  BuildMI(MBB, MBB.getFirstTerminator(), DL,
1407  TII->get(AArch64::SEH_EpilogEnd))
1408  .setMIFlag(MachineInstr::FrameDestroy);
1409  return;
1410  }
1411 
1412  NumBytes -= PrologueSaveSize;
1413  assert(NumBytes >= 0 && "Negative stack allocation size!?");
1414 
1415  if (!hasFP(MF)) {
1416  bool RedZone = canUseRedZone(MF);
1417  // If this was a redzone leaf function, we don't need to restore the
1418  // stack pointer (but we may need to pop stack args for fastcc).
1419  if (RedZone && AfterCSRPopSize == 0)
1420  return;
1421 
1422  bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1423  int StackRestoreBytes = RedZone ? 0 : NumBytes;
1424  if (NoCalleeSaveRestore)
1425  StackRestoreBytes += AfterCSRPopSize;
1426 
1427  // If we were able to combine the local stack pop with the argument pop,
1428  // then we're done.
1429  bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1430 
1431  // If we're done after this, make sure to help the load store optimizer.
1432  if (Done)
1433  adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1434 
1435  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1436  StackRestoreBytes, TII, MachineInstr::FrameDestroy, false,
1437  NeedsWinCFI, &HasWinCFI);
1438  if (Done) {
1439  if (NeedsWinCFI) {
1440  HasWinCFI = true;
1441  BuildMI(MBB, MBB.getFirstTerminator(), DL,
1442  TII->get(AArch64::SEH_EpilogEnd))
1443  .setMIFlag(MachineInstr::FrameDestroy);
1444  }
1445  return;
1446  }
1447 
1448  NumBytes = 0;
1449  }
1450 
1451  // Restore the original stack pointer.
1452  // FIXME: Rather than doing the math here, we should instead just use
1453  // non-post-indexed loads for the restores if we aren't actually going to
1454  // be able to save any instructions.
1455  if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned()))
1456  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1457  -AFI->getCalleeSavedStackSize() + 16, TII,
1458  MachineInstr::FrameDestroy, false, NeedsWinCFI);
1459  else if (NumBytes)
1460  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
1461  MachineInstr::FrameDestroy, false, NeedsWinCFI);
1462 
1463  // This must be placed after the callee-save restore code because that code
1464  // assumes the SP is at the same location as it was after the callee-save save
1465  // code in the prologue.
1466  if (AfterCSRPopSize) {
1467  // Find an insertion point for the first ldp so that it goes before the
1468  // shadow call stack epilog instruction. This ensures that the restore of
1469  // lr from x18 is placed after the restore from sp.
1470  auto FirstSPPopI = MBB.getFirstTerminator();
1471  while (FirstSPPopI != Begin) {
1472  auto Prev = std::prev(FirstSPPopI);
1473  if (Prev->getOpcode() != AArch64::LDRXpre ||
1474  Prev->getOperand(0).getReg() == AArch64::SP)
1475  break;
1476  FirstSPPopI = Prev;
1477  }
1478 
1479  adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1480 
1481  emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1482  AfterCSRPopSize, TII, MachineInstr::FrameDestroy, false,
1483  NeedsWinCFI, &HasWinCFI);
1484  }
1485  if (NeedsWinCFI && HasWinCFI)
1486  BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1487  .setMIFlag(MachineInstr::FrameDestroy);
1488 
1489  MF.setHasWinCFI(HasWinCFI);
1490 }
1491 
1492 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1493 /// debug info. It's the same as what we use for resolving the code-gen
1494 /// references for now. FIXME: This can go wrong when references are
1495 /// SP-relative and simple call frames aren't used.
1497  int FI,
1498  unsigned &FrameReg) const {
1500  MF, FI, FrameReg,
1501  /*PreferFP=*/
1502  MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
1503  /*ForSimm=*/false);
1504 }
1505 
1507  const MachineFunction &MF, int FI) const {
1508  return getSEHFrameIndexOffset(MF, FI);
1509 }
1510 
1511 static int getFPOffset(const MachineFunction &MF, int ObjectOffset) {
1512  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1513  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1514  bool IsWin64 =
1515  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1516  unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1517  return ObjectOffset + FixedObject + 16;
1518 }
1519 
1520 static int getStackOffset(const MachineFunction &MF, int ObjectOffset) {
1521  const auto &MFI = MF.getFrameInfo();
1522  return ObjectOffset + MFI.getStackSize();
1523 }
1524 
1526  int FI) const {
1527  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1528  MF.getSubtarget().getRegisterInfo());
1529  int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
1530  return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
1531  ? getFPOffset(MF, ObjectOffset)
1532  : getStackOffset(MF, ObjectOffset);
1533 }
1534 
1536  int FI, unsigned &FrameReg,
1537  bool PreferFP,
1538  bool ForSimm) const {
1539  const auto &MFI = MF.getFrameInfo();
1540  int ObjectOffset = MFI.getObjectOffset(FI);
1541  bool isFixed = MFI.isFixedObjectIndex(FI);
1542  return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, FrameReg,
1543  PreferFP, ForSimm);
1544 }
1545 
1547  const MachineFunction &MF, int ObjectOffset, bool isFixed,
1548  unsigned &FrameReg, bool PreferFP, bool ForSimm) const {
1549  const auto &MFI = MF.getFrameInfo();
1550  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1551  MF.getSubtarget().getRegisterInfo());
1552  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1553  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1554 
1555  int FPOffset = getFPOffset(MF, ObjectOffset);
1556  int Offset = getStackOffset(MF, ObjectOffset);
1557  bool isCSR =
1558  !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize());
1559 
1560  // Use frame pointer to reference fixed objects. Use it for locals if
1561  // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1562  // reliable as a base). Make sure useFPForScavengingIndex() does the
1563  // right thing for the emergency spill slot.
1564  bool UseFP = false;
1565  if (AFI->hasStackFrame()) {
1566  // Note: Keeping the following as multiple 'if' statements rather than
1567  // merging to a single expression for readability.
1568  //
1569  // Argument access should always use the FP.
1570  if (isFixed) {
1571  UseFP = hasFP(MF);
1572  } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1573  // References to the CSR area must use FP if we're re-aligning the stack
1574  // since the dynamically-sized alignment padding is between the SP/BP and
1575  // the CSR area.
1576  assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1577  UseFP = true;
1578  } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1579  // If the FPOffset is negative and we're producing a signed immediate, we
1580  // have to keep in mind that the available offset range for negative
1581  // offsets is smaller than for positive ones. If an offset is available
1582  // via the FP and the SP, use whichever is closest.
1583  bool FPOffsetFits = !ForSimm || FPOffset >= -256;
1584  PreferFP |= Offset > -FPOffset;
1585 
1586  if (MFI.hasVarSizedObjects()) {
1587  // If we have variable sized objects, we can use either FP or BP, as the
1588  // SP offset is unknown. We can use the base pointer if we have one and
1589  // FP is not preferred. If not, we're stuck with using FP.
1590  bool CanUseBP = RegInfo->hasBasePointer(MF);
1591  if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1592  UseFP = PreferFP;
1593  else if (!CanUseBP) // Can't use BP. Forced to use FP.
1594  UseFP = true;
1595  // else we can use BP and FP, but the offset from FP won't fit.
1596  // That will make us scavenge registers which we can probably avoid by
1597  // using BP. If it won't fit for BP either, we'll scavenge anyway.
1598  } else if (FPOffset >= 0) {
1599  // Use SP or FP, whichever gives us the best chance of the offset
1600  // being in range for direct access. If the FPOffset is positive,
1601  // that'll always be best, as the SP will be even further away.
1602  UseFP = true;
1603  } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1604  // Funclets access the locals contained in the parent's stack frame
1605  // via the frame pointer, so we have to use the FP in the parent
1606  // function.
1607  (void) Subtarget;
1608  assert(
1609  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
1610  "Funclets should only be present on Win64");
1611  UseFP = true;
1612  } else {
1613  // We have the choice between FP and (SP or BP).
1614  if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1615  UseFP = true;
1616  }
1617  }
1618  }
1619 
1620  assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1621  "In the presence of dynamic stack pointer realignment, "
1622  "non-argument/CSR objects cannot be accessed through the frame pointer");
1623 
1624  if (UseFP) {
1625  FrameReg = RegInfo->getFrameRegister(MF);
1626  return FPOffset;
1627  }
1628 
1629  // Use the base pointer if we have one.
1630  if (RegInfo->hasBasePointer(MF))
1631  FrameReg = RegInfo->getBaseRegister();
1632  else {
1633  assert(!MFI.hasVarSizedObjects() &&
1634  "Can't use SP when we have var sized objects.");
1635  FrameReg = AArch64::SP;
1636  // If we're using the red zone for this function, the SP won't actually
1637  // be adjusted, so the offsets will be negative. They're also all
1638  // within range of the signed 9-bit immediate instructions.
1639  if (canUseRedZone(MF))
1640  Offset -= AFI->getLocalStackSize();
1641  }
1642 
1643  return Offset;
1644 }
1645 
1646 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1647  // Do not set a kill flag on values that are also marked as live-in. This
1648  // happens with the @llvm-returnaddress intrinsic and with arguments passed in
1649  // callee saved registers.
1650  // Omitting the kill flags is conservatively correct even if the live-in
1651  // is not used after all.
1652  bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1653  return getKillRegState(!IsLiveIn);
1654 }
1655 
1657  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1659  return Subtarget.isTargetMachO() &&
1660  !(Subtarget.getTargetLowering()->supportSwiftError() &&
1661  Attrs.hasAttrSomewhere(Attribute::SwiftError));
1662 }
1663 
1664 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
1665  bool NeedsWinCFI) {
1666  // If we are generating register pairs for a Windows function that requires
1667  // EH support, then pair consecutive registers only. There are no unwind
1668  // opcodes for saves/restores of non-consectuve register pairs.
1669  // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x.
1670  // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1671 
1672  // TODO: LR can be paired with any register. We don't support this yet in
1673  // the MCLayer. We need to add support for the save_lrpair unwind code.
1674  if (!NeedsWinCFI)
1675  return false;
1676  if (Reg2 == Reg1 + 1)
1677  return false;
1678  return true;
1679 }
1680 
1681 namespace {
1682 
1683 struct RegPairInfo {
1684  unsigned Reg1 = AArch64::NoRegister;
1685  unsigned Reg2 = AArch64::NoRegister;
1686  int FrameIdx;
1687  int Offset;
1688  enum RegType { GPR, FPR64, FPR128 } Type;
1689 
1690  RegPairInfo() = default;
1691 
1692  bool isPaired() const { return Reg2 != AArch64::NoRegister; }
1693 };
1694 
1695 } // end anonymous namespace
1696 
1698  MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
1700  bool &NeedShadowCallStackProlog) {
1701 
1702  if (CSI.empty())
1703  return;
1704 
1705  bool NeedsWinCFI = needsWinCFI(MF);
1707  MachineFrameInfo &MFI = MF.getFrameInfo();
1709  unsigned Count = CSI.size();
1710  (void)CC;
1711  // MachO's compact unwind format relies on all registers being stored in
1712  // pairs.
1714  CC == CallingConv::PreserveMost ||
1715  (Count & 1) == 0) &&
1716  "Odd number of callee-saved regs to spill!");
1717  int Offset = AFI->getCalleeSavedStackSize();
1718  // On Linux, we will have either one or zero non-paired register. On Windows
1719  // with CFI, we can have multiple unpaired registers in order to utilize the
1720  // available unwind codes. This flag assures that the alignment fixup is done
1721  // only once, as intened.
1722  bool FixupDone = false;
1723  for (unsigned i = 0; i < Count; ++i) {
1724  RegPairInfo RPI;
1725  RPI.Reg1 = CSI[i].getReg();
1726 
1727  if (AArch64::GPR64RegClass.contains(RPI.Reg1))
1728  RPI.Type = RegPairInfo::GPR;
1729  else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
1730  RPI.Type = RegPairInfo::FPR64;
1731  else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
1732  RPI.Type = RegPairInfo::FPR128;
1733  else
1734  llvm_unreachable("Unsupported register class.");
1735 
1736  // Add the next reg to the pair if it is in the same register class.
1737  if (i + 1 < Count) {
1738  unsigned NextReg = CSI[i + 1].getReg();
1739  switch (RPI.Type) {
1740  case RegPairInfo::GPR:
1741  if (AArch64::GPR64RegClass.contains(NextReg) &&
1742  !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
1743  RPI.Reg2 = NextReg;
1744  break;
1745  case RegPairInfo::FPR64:
1746  if (AArch64::FPR64RegClass.contains(NextReg) &&
1747  !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
1748  RPI.Reg2 = NextReg;
1749  break;
1750  case RegPairInfo::FPR128:
1751  if (AArch64::FPR128RegClass.contains(NextReg))
1752  RPI.Reg2 = NextReg;
1753  break;
1754  }
1755  }
1756 
1757  // If either of the registers to be saved is the lr register, it means that
1758  // we also need to save lr in the shadow call stack.
1759  if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
1760  MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
1762  report_fatal_error("Must reserve x18 to use shadow call stack");
1763  NeedShadowCallStackProlog = true;
1764  }
1765 
1766  // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
1767  // list to come in sorted by frame index so that we can issue the store
1768  // pair instructions directly. Assert if we see anything otherwise.
1769  //
1770  // The order of the registers in the list is controlled by
1771  // getCalleeSavedRegs(), so they will always be in-order, as well.
1772  assert((!RPI.isPaired() ||
1773  (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
1774  "Out of order callee saved regs!");
1775 
1776  // MachO's compact unwind format relies on all registers being stored in
1777  // adjacent register pairs.
1779  CC == CallingConv::PreserveMost ||
1780  (RPI.isPaired() &&
1781  ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
1782  RPI.Reg1 + 1 == RPI.Reg2))) &&
1783  "Callee-save registers not saved as adjacent register pair!");
1784 
1785  RPI.FrameIdx = CSI[i].getFrameIdx();
1786 
1787  int Scale = RPI.Type == RegPairInfo::FPR128 ? 16 : 8;
1788  Offset -= RPI.isPaired() ? 2 * Scale : Scale;
1789 
1790  // Round up size of non-pair to pair size if we need to pad the
1791  // callee-save area to ensure 16-byte alignment.
1792  if (AFI->hasCalleeSaveStackFreeSpace() && !FixupDone &&
1793  RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired()) {
1794  FixupDone = true;
1795  Offset -= 8;
1796  assert(Offset % 16 == 0);
1797  assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
1798  MFI.setObjectAlignment(RPI.FrameIdx, 16);
1799  }
1800 
1801  assert(Offset % Scale == 0);
1802  RPI.Offset = Offset / Scale;
1803  assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
1804  "Offset out of bounds for LDP/STP immediate");
1805 
1806  RegPairs.push_back(RPI);
1807  if (RPI.isPaired())
1808  ++i;
1809  }
1810 }
1811 
1814  const std::vector<CalleeSavedInfo> &CSI,
1815  const TargetRegisterInfo *TRI) const {
1816  MachineFunction &MF = *MBB.getParent();
1817  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1818  bool NeedsWinCFI = needsWinCFI(MF);
1819  DebugLoc DL;
1820  SmallVector<RegPairInfo, 8> RegPairs;
1821 
1822  bool NeedShadowCallStackProlog = false;
1823  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1824  NeedShadowCallStackProlog);
1825  const MachineRegisterInfo &MRI = MF.getRegInfo();
1826 
1827  if (NeedShadowCallStackProlog) {
1828  // Shadow call stack prolog: str x30, [x18], #8
1829  BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
1830  .addReg(AArch64::X18, RegState::Define)
1831  .addReg(AArch64::LR)
1832  .addReg(AArch64::X18)
1833  .addImm(8)
1835 
1836  if (NeedsWinCFI)
1837  BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
1838  .setMIFlag(MachineInstr::FrameSetup);
1839 
1840  if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
1841  // Emit a CFI instruction that causes 8 to be subtracted from the value of
1842  // x18 when unwinding past this frame.
1843  static const char CFIInst[] = {
1844  dwarf::DW_CFA_val_expression,
1845  18, // register
1846  2, // length
1847  static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
1848  static_cast<char>(-8) & 0x7f, // addend (sleb128)
1849  };
1850  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
1851  nullptr, StringRef(CFIInst, sizeof(CFIInst))));
1852  BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
1853  .addCFIIndex(CFIIndex)
1855  }
1856 
1857  // This instruction also makes x18 live-in to the entry block.
1858  MBB.addLiveIn(AArch64::X18);
1859  }
1860 
1861  for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
1862  ++RPII) {
1863  RegPairInfo RPI = *RPII;
1864  unsigned Reg1 = RPI.Reg1;
1865  unsigned Reg2 = RPI.Reg2;
1866  unsigned StrOpc;
1867 
1868  // Issue sequence of spills for cs regs. The first spill may be converted
1869  // to a pre-decrement store later by emitPrologue if the callee-save stack
1870  // area allocation can't be combined with the local stack area allocation.
1871  // For example:
1872  // stp x22, x21, [sp, #0] // addImm(+0)
1873  // stp x20, x19, [sp, #16] // addImm(+2)
1874  // stp fp, lr, [sp, #32] // addImm(+4)
1875  // Rationale: This sequence saves uop updates compared to a sequence of
1876  // pre-increment spills like stp xi,xj,[sp,#-16]!
1877  // Note: Similar rationale and sequence for restores in epilog.
1878  unsigned Size, Align;
1879  switch (RPI.Type) {
1880  case RegPairInfo::GPR:
1881  StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
1882  Size = 8;
1883  Align = 8;
1884  break;
1885  case RegPairInfo::FPR64:
1886  StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
1887  Size = 8;
1888  Align = 8;
1889  break;
1890  case RegPairInfo::FPR128:
1891  StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
1892  Size = 16;
1893  Align = 16;
1894  break;
1895  }
1896  LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
1897  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1898  dbgs() << ") -> fi#(" << RPI.FrameIdx;
1899  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1900  dbgs() << ")\n");
1901 
1902  assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
1903  "Windows unwdinding requires a consecutive (FP,LR) pair");
1904  // Windows unwind codes require consecutive registers if registers are
1905  // paired. Make the switch here, so that the code below will save (x,x+1)
1906  // and not (x+1,x).
1907  unsigned FrameIdxReg1 = RPI.FrameIdx;
1908  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
1909  if (NeedsWinCFI && RPI.isPaired()) {
1910  std::swap(Reg1, Reg2);
1911  std::swap(FrameIdxReg1, FrameIdxReg2);
1912  }
1913  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
1914  if (!MRI.isReserved(Reg1))
1915  MBB.addLiveIn(Reg1);
1916  if (RPI.isPaired()) {
1917  if (!MRI.isReserved(Reg2))
1918  MBB.addLiveIn(Reg2);
1919  MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
1921  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
1922  MachineMemOperand::MOStore, Size, Align));
1923  }
1924  MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
1925  .addReg(AArch64::SP)
1926  .addImm(RPI.Offset) // [sp, #offset*scale],
1927  // where factor*scale is implicit
1930  MachinePointerInfo::getFixedStack(MF,FrameIdxReg1),
1931  MachineMemOperand::MOStore, Size, Align));
1932  if (NeedsWinCFI)
1934 
1935  }
1936  return true;
1937 }
1938 
1941  std::vector<CalleeSavedInfo> &CSI,
1942  const TargetRegisterInfo *TRI) const {
1943  MachineFunction &MF = *MBB.getParent();
1944  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1945  DebugLoc DL;
1946  SmallVector<RegPairInfo, 8> RegPairs;
1947  bool NeedsWinCFI = needsWinCFI(MF);
1948 
1949  if (MI != MBB.end())
1950  DL = MI->getDebugLoc();
1951 
1952  bool NeedShadowCallStackProlog = false;
1953  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1954  NeedShadowCallStackProlog);
1955 
1956  auto EmitMI = [&](const RegPairInfo &RPI) {
1957  unsigned Reg1 = RPI.Reg1;
1958  unsigned Reg2 = RPI.Reg2;
1959 
1960  // Issue sequence of restores for cs regs. The last restore may be converted
1961  // to a post-increment load later by emitEpilogue if the callee-save stack
1962  // area allocation can't be combined with the local stack area allocation.
1963  // For example:
1964  // ldp fp, lr, [sp, #32] // addImm(+4)
1965  // ldp x20, x19, [sp, #16] // addImm(+2)
1966  // ldp x22, x21, [sp, #0] // addImm(+0)
1967  // Note: see comment in spillCalleeSavedRegisters()
1968  unsigned LdrOpc;
1969  unsigned Size, Align;
1970  switch (RPI.Type) {
1971  case RegPairInfo::GPR:
1972  LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
1973  Size = 8;
1974  Align = 8;
1975  break;
1976  case RegPairInfo::FPR64:
1977  LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
1978  Size = 8;
1979  Align = 8;
1980  break;
1981  case RegPairInfo::FPR128:
1982  LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
1983  Size = 16;
1984  Align = 16;
1985  break;
1986  }
1987  LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
1988  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1989  dbgs() << ") -> fi#(" << RPI.FrameIdx;
1990  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1991  dbgs() << ")\n");
1992 
1993  // Windows unwind codes require consecutive registers if registers are
1994  // paired. Make the switch here, so that the code below will save (x,x+1)
1995  // and not (x+1,x).
1996  unsigned FrameIdxReg1 = RPI.FrameIdx;
1997  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
1998  if (NeedsWinCFI && RPI.isPaired()) {
1999  std::swap(Reg1, Reg2);
2000  std::swap(FrameIdxReg1, FrameIdxReg2);
2001  }
2002  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
2003  if (RPI.isPaired()) {
2004  MIB.addReg(Reg2, getDefRegState(true));
2006  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2007  MachineMemOperand::MOLoad, Size, Align));
2008  }
2009  MIB.addReg(Reg1, getDefRegState(true))
2010  .addReg(AArch64::SP)
2011  .addImm(RPI.Offset) // [sp, #offset*scale]
2012  // where factor*scale is implicit
2015  MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2016  MachineMemOperand::MOLoad, Size, Align));
2017  if (NeedsWinCFI)
2019  };
2021  for (const RegPairInfo &RPI : reverse(RegPairs))
2022  EmitMI(RPI);
2023  else
2024  for (const RegPairInfo &RPI : RegPairs)
2025  EmitMI(RPI);
2026 
2027  if (NeedShadowCallStackProlog) {
2028  // Shadow call stack epilog: ldr x30, [x18, #-8]!
2029  BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
2030  .addReg(AArch64::X18, RegState::Define)
2031  .addReg(AArch64::LR, RegState::Define)
2032  .addReg(AArch64::X18)
2033  .addImm(-8)
2035  }
2036 
2037  return true;
2038 }
2039 
2041  BitVector &SavedRegs,
2042  RegScavenger *RS) const {
2043  // All calls are tail calls in GHC calling conv, and functions have no
2044  // prologue/epilogue.
2046  return;
2047 
2048  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2049  const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
2050  MF.getSubtarget().getRegisterInfo());
2052  unsigned UnspilledCSGPR = AArch64::NoRegister;
2053  unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2054 
2055  MachineFrameInfo &MFI = MF.getFrameInfo();
2056  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2057 
2058  unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
2059  ? RegInfo->getBaseRegister()
2060  : (unsigned)AArch64::NoRegister;
2061 
2062  unsigned ExtraCSSpill = 0;
2063  // Figure out which callee-saved registers to save/restore.
2064  for (unsigned i = 0; CSRegs[i]; ++i) {
2065  const unsigned Reg = CSRegs[i];
2066 
2067  // Add the base pointer register to SavedRegs if it is callee-save.
2068  if (Reg == BasePointerReg)
2069  SavedRegs.set(Reg);
2070 
2071  bool RegUsed = SavedRegs.test(Reg);
2072  unsigned PairedReg = CSRegs[i ^ 1];
2073  if (!RegUsed) {
2074  if (AArch64::GPR64RegClass.contains(Reg) &&
2075  !RegInfo->isReservedReg(MF, Reg)) {
2076  UnspilledCSGPR = Reg;
2077  UnspilledCSGPRPaired = PairedReg;
2078  }
2079  continue;
2080  }
2081 
2082  // MachO's compact unwind format relies on all registers being stored in
2083  // pairs.
2084  // FIXME: the usual format is actually better if unwinding isn't needed.
2085  if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
2086  !SavedRegs.test(PairedReg)) {
2087  SavedRegs.set(PairedReg);
2088  if (AArch64::GPR64RegClass.contains(PairedReg) &&
2089  !RegInfo->isReservedReg(MF, PairedReg))
2090  ExtraCSSpill = PairedReg;
2091  }
2092  }
2093 
2094  // Calculates the callee saved stack size.
2095  unsigned CSStackSize = 0;
2097  const MachineRegisterInfo &MRI = MF.getRegInfo();
2098  for (unsigned Reg : SavedRegs.set_bits())
2099  CSStackSize += TRI->getRegSizeInBits(Reg, MRI) / 8;
2100 
2101  // Save number of saved regs, so we can easily update CSStackSize later.
2102  unsigned NumSavedRegs = SavedRegs.count();
2103 
2104  // The frame record needs to be created by saving the appropriate registers
2105  unsigned EstimatedStackSize = MFI.estimateStackSize(MF);
2106  if (hasFP(MF) ||
2107  windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2108  SavedRegs.set(AArch64::FP);
2109  SavedRegs.set(AArch64::LR);
2110  }
2111 
2112  LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
2113  for (unsigned Reg
2114  : SavedRegs.set_bits()) dbgs()
2115  << ' ' << printReg(Reg, RegInfo);
2116  dbgs() << "\n";);
2117 
2118  // If any callee-saved registers are used, the frame cannot be eliminated.
2119  bool CanEliminateFrame = SavedRegs.count() == 0;
2120 
2121  // The CSR spill slots have not been allocated yet, so estimateStackSize
2122  // won't include them.
2123  unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2124  bool BigStack = (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2125  if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2126  AFI->setHasStackFrame(true);
2127 
2128  // Estimate if we might need to scavenge a register at some point in order
2129  // to materialize a stack offset. If so, either spill one additional
2130  // callee-saved register or reserve a special spill slot to facilitate
2131  // register scavenging. If we already spilled an extra callee-saved register
2132  // above to keep the number of spills even, we don't need to do anything else
2133  // here.
2134  if (BigStack) {
2135  if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2136  LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2137  << " to get a scratch register.\n");
2138  SavedRegs.set(UnspilledCSGPR);
2139  // MachO's compact unwind format relies on all registers being stored in
2140  // pairs, so if we need to spill one extra for BigStack, then we need to
2141  // store the pair.
2142  if (produceCompactUnwindFrame(MF))
2143  SavedRegs.set(UnspilledCSGPRPaired);
2144  ExtraCSSpill = UnspilledCSGPRPaired;
2145  }
2146 
2147  // If we didn't find an extra callee-saved register to spill, create
2148  // an emergency spill slot.
2149  if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2150  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2151  const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2152  unsigned Size = TRI->getSpillSize(RC);
2153  unsigned Align = TRI->getSpillAlignment(RC);
2154  int FI = MFI.CreateStackObject(Size, Align, false);
2155  RS->addScavengingFrameIndex(FI);
2156  LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2157  << " as the emergency spill slot.\n");
2158  }
2159  }
2160 
2161  // Adding the size of additional 64bit GPR saves.
2162  CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2163  unsigned AlignedCSStackSize = alignTo(CSStackSize, 16);
2164  LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2165  << EstimatedStackSize + AlignedCSStackSize
2166  << " bytes.\n");
2167 
2168  // Round up to register pair alignment to avoid additional SP adjustment
2169  // instructions.
2170  AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2171  AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2172 }
2173 
2175  const MachineFunction &MF) const {
2176  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2177  return AFI->hasCalleeSaveStackFreeSpace();
2178 }
2179 
2181  MachineFunction &MF, RegScavenger *RS) const {
2182  // If this function isn't doing Win64-style C++ EH, we don't need to do
2183  // anything.
2184  if (!MF.hasEHFunclets())
2185  return;
2186  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2187  MachineFrameInfo &MFI = MF.getFrameInfo();
2188  WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2189 
2190  MachineBasicBlock &MBB = MF.front();
2191  auto MBBI = MBB.begin();
2192  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2193  ++MBBI;
2194 
2195  // Create an UnwindHelp object.
2196  int UnwindHelpFI =
2197  MFI.CreateStackObject(/*size*/8, /*alignment*/16, false);
2198  EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
2199  // We need to store -2 into the UnwindHelp object at the start of the
2200  // function.
2201  DebugLoc DL;
2202  RS->enterBasicBlockEnd(MBB);
2203  RS->backward(std::prev(MBBI));
2204  unsigned DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
2205  assert(DstReg && "There must be a free register after frame setup");
2206  BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
2207  BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
2208  .addReg(DstReg, getKillRegState(true))
2209  .addFrameIndex(UnwindHelpFI)
2210  .addImm(0);
2211 }
2212 
2213 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP before
2214 /// the update. This is easily retrieved as it is exactly the offset that is set
2215 /// in processFunctionBeforeFrameFinalized.
2217  const MachineFunction &MF, int FI, unsigned &FrameReg,
2218  bool IgnoreSPUpdates) const {
2219  const MachineFrameInfo &MFI = MF.getFrameInfo();
2220  LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
2221  << MFI.getObjectOffset(FI) << "\n");
2222  FrameReg = AArch64::SP;
2223  return MFI.getObjectOffset(FI);
2224 }
2225 
2226 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
2227 /// the parent's frame pointer
2229  const MachineFunction &MF) const {
2230  return 0;
2231 }
2232 
2233 /// Funclets only need to account for space for the callee saved registers,
2234 /// as the locals are accounted for in the parent's stack frame.
2236  const MachineFunction &MF) const {
2237  // This is the size of the pushed CSRs.
2238  unsigned CSSize =
2239  MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
2240  // This is the amount of stack a funclet needs to allocate.
2241  return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
2242  getStackAlignment());
2243 }
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
static cl::opt< bool > ReverseCSRRestoreSeq("reverse-csr-restore-seq", cl::desc("reverse the CSR restore sequence"), cl::init(false), cl::Hidden)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand *> MMOs) const
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:590
BitVector & set()
Definition: BitVector.h:397
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasDebugInfo() const
Returns true if valid debug info is present.
static bool ShouldSignWithAKey(MachineFunction &MF)
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static int getStackOffset(const MachineFunction &MF, int ObjectOffset)
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Definition: StringRef.h:167
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
class llvm::RegisterBankInfo GPR
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool &NeedShadowCallStackProlog)
bool cannotEliminateFrame(const MachineFunction &MF) const
LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst)
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset)
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
Definition: MCDwarf.h:507
unsigned Reg
bool test(unsigned Idx) const
Definition: BitVector.h:501
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:323
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
F(f)
MachineModuleInfo & getMMI() const
uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:684
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI)
static MCCFIInstruction createDefCfaOffset(MCSymbol *L, int Offset)
.cfi_def_cfa_offset modifies a rule for computing CFA.
Definition: MCDwarf.h:494
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
bool isCallingConvWin64(CallingConv::ID CC) const
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, unsigned LocalStackSize, bool NeedsWinCFI, bool *HasWinCFI)
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
return AArch64::GPR64RegClass contains(Reg)
static bool isFuncletReturnInstr(const MachineInstr &MI)
static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:119
unsigned getSpillAlignment(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class...
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:721
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
const HexagonInstrInfo * TII
Printable printReg(unsigned Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required, we reserve argument space for call sites in the function immediately on entry to the current function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
This file contains the simple types necessary to represent the attributes associated with functions a...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
void backward()
Update internal register state and move MBB iterator backwards.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
int getDwarfRegNum(unsigned RegNum, bool isEH) const
Map a target register to an equivalent dwarf register number.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
int getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, unsigned &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent&#39;s frame pointer...
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:273
Key
PAL metadata keys.
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
virtual void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, bool KillSrc) const
Emit instructions to copy a pair of physical registers.
unsigned FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
const AArch64RegisterInfo * getRegisterInfo() const override
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
const MCContext & getContext() const
iterator getLastNonDebugInstr()
Returns an iterator to the last non-debug instruction in the basic block, or end().
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:223
bool hasPersonalityFn() const
Check whether this function has a personality function.
Definition: Function.h:726
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
virtual const TargetInstrInfo * getInstrInfo() const
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
bool isXRegisterReserved(size_t i) const
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
unsigned getKillRegState(bool B)
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:117
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
TargetInstrInfo - Interface to description of machine instruction set.
int getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
unsigned getDefRegState(bool B)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
const AArch64TargetLowering * getTargetLowering() const override
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1165
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
static MCCFIInstruction createNegateRAState(MCSymbol *L)
.cfi_negate_ra_state AArch64 negate RA state.
Definition: MCDwarf.h:533
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
This file declares the machine register scavenger class.
MCSymbol * createTempSymbol(bool CanBeUnnamed=true)
Create and return a new assembler temporary symbol with a unique but unspecified name.
Definition: MCContext.cpp:223
unsigned const MachineRegisterInfo * MRI
static MCCFIInstruction createDefCfa(MCSymbol *L, unsigned Register, int Offset)
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it...
Definition: MCDwarf.h:480
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static bool ShouldSignReturnAddress(MachineFunction &MF)
bool hasEHFunclets() const
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size FIXME: The defaults need to be removed once all of the backends/clients are updat...
Definition: DataLayout.cpp:645
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any DBG_VALUE and DBG_LABEL instructions...
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
unsigned getMaxAlignment() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
void setImm(int64_t immVal)
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const MachineBasicBlock & front() const
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const
The memory access writes data.
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:478
size_type count() const
count - Returns the number of bits which are set.
Definition: BitVector.h:172
static void InsertReturnAddressAuth(MachineFunction &MF, MachineBasicBlock &MBB)
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:212
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
unsigned estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling...
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
int64_t getImm() const
const Function & getFunction() const
Return the LLVM function that this machine code represents.
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
Definition: Function.h:594
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
bool isPhysRegUsed(unsigned PhysReg) const
Return true if the specified register is modified or read in this function.
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
int resolveFrameOffsetReference(const MachineFunction &MF, int ObjectOffset, bool isFixed, unsigned &FrameReg, bool PreferFP, bool ForSimm) const
CodeModel::Model getCodeModel() const
Returns the code model.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
static bool needsWinCFI(const MachineFunction &MF)
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static bool windowsRequiresStackProbe(MachineFunction &MF, unsigned StackSizeInBytes)
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register...
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:717
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:256
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
TargetSubtargetInfo - Generic base class for all target subtargets.
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:160
bool isReservedReg(const MachineFunction &MF, unsigned Reg) const
bool isLiveIn(unsigned Reg) const
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
static void adaptForLdStOpt(MachineBasicBlock &MBB, MachineBasicBlock::iterator FirstSPPopI, MachineBasicBlock::iterator LastPopI)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:48
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:223
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
TargetOptions Options
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned char TargetFlags=0) const
#define I(x, y, z)
Definition: MD5.cpp:58
static int getFPOffset(const MachineFunction &MF, int ObjectOffset)
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
uint32_t Size
Definition: Profile.cpp:46
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
static bool produceCompactUnwindFrame(MachineFunction &MF)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
bool hasBasePointer(const MachineFunction &MF) const
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:129
bool needsStackRealignment(const MachineFunction &MF) const
True if storage within the function requires the stack pointer to be aligned more than the normal cal...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1384
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool InProlog=true)
const AArch64InstrInfo * getInstrInfo() const override
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:250
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
Definition: LivePhysRegs.h:79
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:333
bool isMaxCallFrameSizeComputed() const
IRTranslator LLVM IR MI
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
unsigned getRegSizeInBits(const TargetRegisterClass &RC) const
Return the size in bits of a register from class RC.
Register getReg() const
getReg - Returns the register number.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals)
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition: MCDwarf.h:568
int resolveFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg, bool PreferFP, bool ForSimm) const
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects...
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
void setCalleeSavedStackSize(unsigned Size)
This class contains meta information specific to a module.
bool hasCalls() const
Return true if the current function has any function calls.