LLVM  9.0.0svn
AArch64FrameLowering.cpp
Go to the documentation of this file.
1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the AArch64 implementation of TargetFrameLowering class.
11 //
12 // On AArch64, stack frames are structured as follows:
13 //
14 // The stack grows downward.
15 //
16 // All of the individual frame areas on the frame below are optional, i.e. it's
17 // possible to create a function so that the particular area isn't present
18 // in the frame.
19 //
20 // At function entry, the "frame" looks as follows:
21 //
22 // | | Higher address
23 // |-----------------------------------|
24 // | |
25 // | arguments passed on the stack |
26 // | |
27 // |-----------------------------------| <- sp
28 // | | Lower address
29 //
30 //
31 // After the prologue has run, the frame has the following general structure.
32 // Note that this doesn't depict the case where a red-zone is used. Also,
33 // technically the last frame area (VLAs) doesn't get created until in the
34 // main function body, after the prologue is run. However, it's depicted here
35 // for completeness.
36 //
37 // | | Higher address
38 // |-----------------------------------|
39 // | |
40 // | arguments passed on the stack |
41 // | |
42 // |-----------------------------------|
43 // | |
44 // | (Win64 only) varargs from reg |
45 // | |
46 // |-----------------------------------|
47 // | |
48 // | prev_fp, prev_lr |
49 // | (a.k.a. "frame record") |
50 // |-----------------------------------| <- fp(=x29)
51 // | |
52 // | other callee-saved registers |
53 // | |
54 // |-----------------------------------|
55 // |.empty.space.to.make.part.below....|
56 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
57 // |.the.standard.16-byte.alignment....| compile time; if present)
58 // |-----------------------------------|
59 // | |
60 // | local variables of fixed size |
61 // | including spill slots |
62 // |-----------------------------------| <- bp(not defined by ABI,
63 // |.variable-sized.local.variables....| LLVM chooses X19)
64 // |.(VLAs)............................| (size of this area is unknown at
65 // |...................................| compile time)
66 // |-----------------------------------| <- sp
67 // | | Lower address
68 //
69 //
70 // To access the data in a frame, at-compile time, a constant offset must be
71 // computable from one of the pointers (fp, bp, sp) to access it. The size
72 // of the areas with a dotted background cannot be computed at compile-time
73 // if they are present, making it required to have all three of fp, bp and
74 // sp to be set up to be able to access all contents in the frame areas,
75 // assuming all of the frame areas are non-empty.
76 //
77 // For most functions, some of the frame areas are empty. For those functions,
78 // it may not be necessary to set up fp or bp:
79 // * A base pointer is definitely needed when there are both VLAs and local
80 // variables with more-than-default alignment requirements.
81 // * A frame pointer is definitely needed when there are local variables with
82 // more-than-default alignment requirements.
83 //
84 // In some cases when a base pointer is not strictly needed, it is generated
85 // anyway when offsets from the frame pointer to access local variables become
86 // so large that the offset can't be encoded in the immediate fields of loads
87 // or stores.
88 //
89 // FIXME: also explain the redzone concept.
90 // FIXME: also explain the concept of reserved call frames.
91 //
92 //===----------------------------------------------------------------------===//
93 
94 #include "AArch64FrameLowering.h"
95 #include "AArch64InstrInfo.h"
97 #include "AArch64RegisterInfo.h"
98 #include "AArch64Subtarget.h"
99 #include "AArch64TargetMachine.h"
101 #include "llvm/ADT/ScopeExit.h"
102 #include "llvm/ADT/SmallVector.h"
103 #include "llvm/ADT/Statistic.h"
119 #include "llvm/IR/Attributes.h"
120 #include "llvm/IR/CallingConv.h"
121 #include "llvm/IR/DataLayout.h"
122 #include "llvm/IR/DebugLoc.h"
123 #include "llvm/IR/Function.h"
124 #include "llvm/MC/MCAsmInfo.h"
125 #include "llvm/MC/MCDwarf.h"
127 #include "llvm/Support/Debug.h"
129 #include "llvm/Support/MathExtras.h"
133 #include <cassert>
134 #include <cstdint>
135 #include <iterator>
136 #include <vector>
137 
138 using namespace llvm;
139 
140 #define DEBUG_TYPE "frame-info"
141 
142 static cl::opt<bool> EnableRedZone("aarch64-redzone",
143  cl::desc("enable use of redzone on AArch64"),
144  cl::init(false), cl::Hidden);
145 
146 static cl::opt<bool>
147  ReverseCSRRestoreSeq("reverse-csr-restore-seq",
148  cl::desc("reverse the CSR restore sequence"),
149  cl::init(false), cl::Hidden);
150 
151 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
152 
153 /// This is the biggest offset to the stack pointer we can encode in aarch64
154 /// instructions (without using a separate calculation and a temp register).
155 /// Note that the exception here are vector stores/loads which cannot encode any
156 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
157 static const unsigned DefaultSafeSPDisplacement = 255;
158 
159 /// Look at each instruction that references stack frames and return the stack
160 /// size limit beyond which some of these instructions will require a scratch
161 /// register during their expansion later.
163  // FIXME: For now, just conservatively guestimate based on unscaled indexing
164  // range. We'll end up allocating an unnecessary spill slot a lot, but
165  // realistically that's not a big deal at this stage of the game.
166  for (MachineBasicBlock &MBB : MF) {
167  for (MachineInstr &MI : MBB) {
168  if (MI.isDebugInstr() || MI.isPseudo() ||
169  MI.getOpcode() == AArch64::ADDXri ||
170  MI.getOpcode() == AArch64::ADDSXri)
171  continue;
172 
173  for (const MachineOperand &MO : MI.operands()) {
174  if (!MO.isFI())
175  continue;
176 
177  int Offset = 0;
178  if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
180  return 0;
181  }
182  }
183  }
185 }
186 
188  if (!EnableRedZone)
189  return false;
190  // Don't use the red zone if the function explicitly asks us not to.
191  // This is typically used for kernel code.
192  if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
193  return false;
194 
195  const MachineFrameInfo &MFI = MF.getFrameInfo();
197  unsigned NumBytes = AFI->getLocalStackSize();
198 
199  return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
200 }
201 
202 /// hasFP - Return true if the specified function should have a dedicated frame
203 /// pointer register.
205  const MachineFrameInfo &MFI = MF.getFrameInfo();
206  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
207  // Win64 EH requires a frame pointer if funclets are present, as the locals
208  // are accessed off the frame pointer in both the parent function and the
209  // funclets.
210  if (MF.hasEHFunclets())
211  return true;
212  // Retain behavior of always omitting the FP for leaf functions when possible.
213  if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
214  return true;
215  if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
216  MFI.hasStackMap() || MFI.hasPatchPoint() ||
217  RegInfo->needsStackRealignment(MF))
218  return true;
219  // With large callframes around we may need to use FP to access the scavenging
220  // emergency spillslot.
221  //
222  // Unfortunately some calls to hasFP() like machine verifier ->
223  // getReservedReg() -> hasFP in the middle of global isel are too early
224  // to know the max call frame size. Hopefully conservatively returning "true"
225  // in those cases is fine.
226  // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
227  if (!MFI.isMaxCallFrameSizeComputed() ||
229  return true;
230 
231  // Win64 SEH requires frame pointer if funclets are present.
232  if (MF.hasLocalEscape())
233  return true;
234 
235  return false;
236 }
237 
238 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
239 /// not required, we reserve argument space for call sites in the function
240 /// immediately on entry to the current function. This eliminates the need for
241 /// add/sub sp brackets around call sites. Returns true if the call frame is
242 /// included as part of the stack frame.
243 bool
245  return !MF.getFrameInfo().hasVarSizedObjects();
246 }
247 
251  const AArch64InstrInfo *TII =
252  static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
253  DebugLoc DL = I->getDebugLoc();
254  unsigned Opc = I->getOpcode();
255  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
256  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
257 
259  if (!TFI->hasReservedCallFrame(MF)) {
260  unsigned Align = getStackAlignment();
261 
262  int64_t Amount = I->getOperand(0).getImm();
263  Amount = alignTo(Amount, Align);
264  if (!IsDestroy)
265  Amount = -Amount;
266 
267  // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
268  // doesn't have to pop anything), then the first operand will be zero too so
269  // this adjustment is a no-op.
270  if (CalleePopAmount == 0) {
271  // FIXME: in-function stack adjustment for calls is limited to 24-bits
272  // because there's no guaranteed temporary register available.
273  //
274  // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
275  // 1) For offset <= 12-bit, we use LSL #0
276  // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
277  // LSL #0, and the other uses LSL #12.
278  //
279  // Most call frames will be allocated at the start of a function so
280  // this is OK, but it is a limitation that needs dealing with.
281  assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
282  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
283  }
284  } else if (CalleePopAmount != 0) {
285  // If the calling convention demands that the callee pops arguments from the
286  // stack, we want to add it back if we have a reserved call frame.
287  assert(CalleePopAmount < 0xffffff && "call frame too large");
288  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
289  TII);
290  }
291  return MBB.erase(I);
292 }
293 
295  // The function should be signed in the following situations:
296  // - sign-return-address=all
297  // - sign-return-address=non-leaf and the functions spills the LR
298 
299  const Function &F = MF.getFunction();
300  if (!F.hasFnAttribute("sign-return-address"))
301  return false;
302 
303  StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
304  if (Scope.equals("none"))
305  return false;
306 
307  if (Scope.equals("all"))
308  return true;
309 
310  assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
311 
312  for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
313  if (Info.getReg() == AArch64::LR)
314  return true;
315 
316  return false;
317 }
318 
321  MachineFunction &MF = *MBB.getParent();
322  MachineFrameInfo &MFI = MF.getFrameInfo();
323  const TargetSubtargetInfo &STI = MF.getSubtarget();
324  const MCRegisterInfo *MRI = STI.getRegisterInfo();
325  const TargetInstrInfo *TII = STI.getInstrInfo();
326  DebugLoc DL = MBB.findDebugLoc(MBBI);
327 
328  // Add callee saved registers to move list.
329  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
330  if (CSI.empty())
331  return;
332 
333  for (const auto &Info : CSI) {
334  unsigned Reg = Info.getReg();
335  int64_t Offset =
336  MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
337  unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
338  unsigned CFIIndex = MF.addFrameInst(
339  MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
340  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
341  .addCFIIndex(CFIIndex)
343  }
344 }
345 
346 // Find a scratch register that we can use at the start of the prologue to
347 // re-align the stack pointer. We avoid using callee-save registers since they
348 // may appear to be free when this is called from canUseAsPrologue (during
349 // shrink wrapping), but then no longer be free when this is called from
350 // emitPrologue.
351 //
352 // FIXME: This is a bit conservative, since in the above case we could use one
353 // of the callee-save registers as a scratch temp to re-align the stack pointer,
354 // but we would then have to make sure that we were in fact saving at least one
355 // callee-save register in the prologue, which is additional complexity that
356 // doesn't seem worth the benefit.
358  MachineFunction *MF = MBB->getParent();
359 
360  // If MBB is an entry block, use X9 as the scratch register
361  if (&MF->front() == MBB)
362  return AArch64::X9;
363 
364  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
365  const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
366  LivePhysRegs LiveRegs(TRI);
367  LiveRegs.addLiveIns(*MBB);
368 
369  // Mark callee saved registers as used so we will not choose them.
370  const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
371  for (unsigned i = 0; CSRegs[i]; ++i)
372  LiveRegs.addReg(CSRegs[i]);
373 
374  // Prefer X9 since it was historically used for the prologue scratch reg.
375  const MachineRegisterInfo &MRI = MF->getRegInfo();
376  if (LiveRegs.available(MRI, AArch64::X9))
377  return AArch64::X9;
378 
379  for (unsigned Reg : AArch64::GPR64RegClass) {
380  if (LiveRegs.available(MRI, Reg))
381  return Reg;
382  }
383  return AArch64::NoRegister;
384 }
385 
387  const MachineBasicBlock &MBB) const {
388  const MachineFunction *MF = MBB.getParent();
389  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
390  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
391  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
392 
393  // Don't need a scratch register if we're not going to re-align the stack.
394  if (!RegInfo->needsStackRealignment(*MF))
395  return true;
396  // Otherwise, we can use any block as long as it has a scratch register
397  // available.
398  return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
399 }
400 
402  unsigned StackSizeInBytes) {
403  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
404  if (!Subtarget.isTargetWindows())
405  return false;
406  const Function &F = MF.getFunction();
407  // TODO: When implementing stack protectors, take that into account
408  // for the probe threshold.
409  unsigned StackProbeSize = 4096;
410  if (F.hasFnAttribute("stack-probe-size"))
411  F.getFnAttribute("stack-probe-size")
413  .getAsInteger(0, StackProbeSize);
414  return (StackSizeInBytes >= StackProbeSize) &&
415  !F.hasFnAttribute("no-stack-arg-probe");
416 }
417 
418 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
419  MachineFunction &MF, unsigned StackBumpBytes) const {
421  const MachineFrameInfo &MFI = MF.getFrameInfo();
422  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
423  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
424 
425  if (AFI->getLocalStackSize() == 0)
426  return false;
427 
428  // 512 is the maximum immediate for stp/ldp that will be used for
429  // callee-save save/restores
430  if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
431  return false;
432 
433  if (MFI.hasVarSizedObjects())
434  return false;
435 
436  if (RegInfo->needsStackRealignment(MF))
437  return false;
438 
439  // This isn't strictly necessary, but it simplifies things a bit since the
440  // current RedZone handling code assumes the SP is adjusted by the
441  // callee-save save/restore code.
442  if (canUseRedZone(MF))
443  return false;
444 
445  return true;
446 }
447 
448 // Given a load or a store instruction, generate an appropriate unwinding SEH
449 // code on Windows.
451  const TargetInstrInfo &TII,
453  unsigned Opc = MBBI->getOpcode();
454  MachineBasicBlock *MBB = MBBI->getParent();
455  MachineFunction &MF = *MBB->getParent();
456  DebugLoc DL = MBBI->getDebugLoc();
457  unsigned ImmIdx = MBBI->getNumOperands() - 1;
458  int Imm = MBBI->getOperand(ImmIdx).getImm();
460  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
461  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
462 
463  switch (Opc) {
464  default:
465  llvm_unreachable("No SEH Opcode for this instruction");
466  case AArch64::LDPDpost:
467  Imm = -Imm;
469  case AArch64::STPDpre: {
470  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
471  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
472  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
473  .addImm(Reg0)
474  .addImm(Reg1)
475  .addImm(Imm * 8)
476  .setMIFlag(Flag);
477  break;
478  }
479  case AArch64::LDPXpost:
480  Imm = -Imm;
482  case AArch64::STPXpre: {
483  unsigned Reg0 = MBBI->getOperand(1).getReg();
484  unsigned Reg1 = MBBI->getOperand(2).getReg();
485  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
486  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
487  .addImm(Imm * 8)
488  .setMIFlag(Flag);
489  else
490  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
491  .addImm(RegInfo->getSEHRegNum(Reg0))
492  .addImm(RegInfo->getSEHRegNum(Reg1))
493  .addImm(Imm * 8)
494  .setMIFlag(Flag);
495  break;
496  }
497  case AArch64::LDRDpost:
498  Imm = -Imm;
500  case AArch64::STRDpre: {
501  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
502  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
503  .addImm(Reg)
504  .addImm(Imm)
505  .setMIFlag(Flag);
506  break;
507  }
508  case AArch64::LDRXpost:
509  Imm = -Imm;
511  case AArch64::STRXpre: {
512  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
513  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
514  .addImm(Reg)
515  .addImm(Imm)
516  .setMIFlag(Flag);
517  break;
518  }
519  case AArch64::STPDi:
520  case AArch64::LDPDi: {
521  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
522  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
523  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
524  .addImm(Reg0)
525  .addImm(Reg1)
526  .addImm(Imm * 8)
527  .setMIFlag(Flag);
528  break;
529  }
530  case AArch64::STPXi:
531  case AArch64::LDPXi: {
532  unsigned Reg0 = MBBI->getOperand(0).getReg();
533  unsigned Reg1 = MBBI->getOperand(1).getReg();
534  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
535  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
536  .addImm(Imm * 8)
537  .setMIFlag(Flag);
538  else
539  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
540  .addImm(RegInfo->getSEHRegNum(Reg0))
541  .addImm(RegInfo->getSEHRegNum(Reg1))
542  .addImm(Imm * 8)
543  .setMIFlag(Flag);
544  break;
545  }
546  case AArch64::STRXui:
547  case AArch64::LDRXui: {
548  int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
549  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
550  .addImm(Reg)
551  .addImm(Imm * 8)
552  .setMIFlag(Flag);
553  break;
554  }
555  case AArch64::STRDui:
556  case AArch64::LDRDui: {
557  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
558  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
559  .addImm(Reg)
560  .addImm(Imm * 8)
561  .setMIFlag(Flag);
562  break;
563  }
564  }
565  auto I = MBB->insertAfter(MBBI, MIB);
566  return I;
567 }
568 
569 // Fix up the SEH opcode associated with the save/restore instruction.
571  unsigned LocalStackSize) {
572  MachineOperand *ImmOpnd = nullptr;
573  unsigned ImmIdx = MBBI->getNumOperands() - 1;
574  switch (MBBI->getOpcode()) {
575  default:
576  llvm_unreachable("Fix the offset in the SEH instruction");
577  case AArch64::SEH_SaveFPLR:
578  case AArch64::SEH_SaveRegP:
579  case AArch64::SEH_SaveReg:
580  case AArch64::SEH_SaveFRegP:
581  case AArch64::SEH_SaveFReg:
582  ImmOpnd = &MBBI->getOperand(ImmIdx);
583  break;
584  }
585  if (ImmOpnd)
586  ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
587 }
588 
589 // Convert callee-save register save/restore instruction to do stack pointer
590 // decrement/increment to allocate/deallocate the callee-save stack area by
591 // converting store/load to use pre/post increment version.
594  const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
595  bool NeedsWinCFI, bool InProlog = true) {
596  // Ignore instructions that do not operate on SP, i.e. shadow call stack
597  // instructions and associated CFI instruction.
598  while (MBBI->getOpcode() == AArch64::STRXpost ||
599  MBBI->getOpcode() == AArch64::LDRXpre ||
600  MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
601  if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
602  assert(MBBI->getOperand(0).getReg() != AArch64::SP);
603  ++MBBI;
604  }
605  unsigned NewOpc;
606  int Scale = 1;
607  switch (MBBI->getOpcode()) {
608  default:
609  llvm_unreachable("Unexpected callee-save save/restore opcode!");
610  case AArch64::STPXi:
611  NewOpc = AArch64::STPXpre;
612  Scale = 8;
613  break;
614  case AArch64::STPDi:
615  NewOpc = AArch64::STPDpre;
616  Scale = 8;
617  break;
618  case AArch64::STPQi:
619  NewOpc = AArch64::STPQpre;
620  Scale = 16;
621  break;
622  case AArch64::STRXui:
623  NewOpc = AArch64::STRXpre;
624  break;
625  case AArch64::STRDui:
626  NewOpc = AArch64::STRDpre;
627  break;
628  case AArch64::STRQui:
629  NewOpc = AArch64::STRQpre;
630  break;
631  case AArch64::LDPXi:
632  NewOpc = AArch64::LDPXpost;
633  Scale = 8;
634  break;
635  case AArch64::LDPDi:
636  NewOpc = AArch64::LDPDpost;
637  Scale = 8;
638  break;
639  case AArch64::LDPQi:
640  NewOpc = AArch64::LDPQpost;
641  Scale = 16;
642  break;
643  case AArch64::LDRXui:
644  NewOpc = AArch64::LDRXpost;
645  break;
646  case AArch64::LDRDui:
647  NewOpc = AArch64::LDRDpost;
648  break;
649  case AArch64::LDRQui:
650  NewOpc = AArch64::LDRQpost;
651  break;
652  }
653  // Get rid of the SEH code associated with the old instruction.
654  if (NeedsWinCFI) {
655  auto SEH = std::next(MBBI);
657  SEH->eraseFromParent();
658  }
659 
660  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
661  MIB.addReg(AArch64::SP, RegState::Define);
662 
663  // Copy all operands other than the immediate offset.
664  unsigned OpndIdx = 0;
665  for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
666  ++OpndIdx)
667  MIB.add(MBBI->getOperand(OpndIdx));
668 
669  assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
670  "Unexpected immediate offset in first/last callee-save save/restore "
671  "instruction!");
672  assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
673  "Unexpected base register in callee-save save/restore instruction!");
674  assert(CSStackSizeInc % Scale == 0);
675  MIB.addImm(CSStackSizeInc / Scale);
676 
677  MIB.setMIFlags(MBBI->getFlags());
678  MIB.setMemRefs(MBBI->memoperands());
679 
680  // Generate a new SEH code that corresponds to the new instruction.
681  if (NeedsWinCFI)
682  InsertSEH(*MIB, *TII,
684 
685  return std::prev(MBB.erase(MBBI));
686 }
687 
688 // Fixup callee-save register save/restore instructions to take into account
689 // combined SP bump by adding the local stack size to the stack offsets.
691  unsigned LocalStackSize,
692  bool NeedsWinCFI) {
694  return;
695 
696  unsigned Opc = MI.getOpcode();
697 
698  // Ignore instructions that do not operate on SP, i.e. shadow call stack
699  // instructions and associated CFI instruction.
700  if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
701  Opc == AArch64::CFI_INSTRUCTION) {
702  if (Opc != AArch64::CFI_INSTRUCTION)
703  assert(MI.getOperand(0).getReg() != AArch64::SP);
704  return;
705  }
706 
707  unsigned Scale;
708  switch (Opc) {
709  case AArch64::STPXi:
710  case AArch64::STRXui:
711  case AArch64::STPDi:
712  case AArch64::STRDui:
713  case AArch64::LDPXi:
714  case AArch64::LDRXui:
715  case AArch64::LDPDi:
716  case AArch64::LDRDui:
717  Scale = 8;
718  break;
719  case AArch64::STPQi:
720  case AArch64::STRQui:
721  case AArch64::LDPQi:
722  case AArch64::LDRQui:
723  Scale = 16;
724  break;
725  default:
726  llvm_unreachable("Unexpected callee-save save/restore opcode!");
727  }
728 
729  unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
730  assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
731  "Unexpected base register in callee-save save/restore instruction!");
732  // Last operand is immediate offset that needs fixing.
733  MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
734  // All generated opcodes have scaled offsets.
735  assert(LocalStackSize % Scale == 0);
736  OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
737 
738  if (NeedsWinCFI) {
739  auto MBBI = std::next(MachineBasicBlock::iterator(MI));
740  assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
742  "Expecting a SEH instruction");
743  fixupSEHOpcode(MBBI, LocalStackSize);
744  }
745 }
746 
748  MachineBasicBlock::iterator FirstSPPopI,
749  MachineBasicBlock::iterator LastPopI) {
750  // Sometimes (when we restore in the same order as we save), we can end up
751  // with code like this:
752  //
753  // ldp x26, x25, [sp]
754  // ldp x24, x23, [sp, #16]
755  // ldp x22, x21, [sp, #32]
756  // ldp x20, x19, [sp, #48]
757  // add sp, sp, #64
758  //
759  // In this case, it is always better to put the first ldp at the end, so
760  // that the load-store optimizer can run and merge the ldp and the add into
761  // a post-index ldp.
762  // If we managed to grab the first pop instruction, move it to the end.
764  MBB.splice(FirstSPPopI, &MBB, LastPopI);
765  // We should end up with something like this now:
766  //
767  // ldp x24, x23, [sp, #16]
768  // ldp x22, x21, [sp, #32]
769  // ldp x20, x19, [sp, #48]
770  // ldp x26, x25, [sp]
771  // add sp, sp, #64
772  //
773  // and the load-store optimizer can merge the last two instructions into:
774  //
775  // ldp x26, x25, [sp], #64
776  //
777 }
778 
780  const Function &F = MF.getFunction();
781  if (!F.hasFnAttribute("sign-return-address-key"))
782  return true;
783 
784  const StringRef Key =
785  F.getFnAttribute("sign-return-address-key").getValueAsString();
786  assert(Key.equals_lower("a_key") || Key.equals_lower("b_key"));
787  return Key.equals_lower("a_key");
788 }
789 
790 static bool needsWinCFI(const MachineFunction &MF) {
791  const Function &F = MF.getFunction();
792  return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
794 }
795 
797  MachineBasicBlock &MBB) const {
798  MachineBasicBlock::iterator MBBI = MBB.begin();
799  const MachineFrameInfo &MFI = MF.getFrameInfo();
800  const Function &F = MF.getFunction();
801  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
802  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
803  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
804  MachineModuleInfo &MMI = MF.getMMI();
806  bool needsFrameMoves = (MMI.hasDebugInfo() || F.needsUnwindTableEntry()) &&
808  bool HasFP = hasFP(MF);
809  bool NeedsWinCFI = needsWinCFI(MF);
810  MF.setHasWinCFI(NeedsWinCFI);
811  bool IsFunclet = MBB.isEHFuncletEntry();
812 
813  // At this point, we're going to decide whether or not the function uses a
814  // redzone. In most cases, the function doesn't have a redzone so let's
815  // assume that's false and set it to true in the case that there's a redzone.
816  AFI->setHasRedZone(false);
817 
818  // Debug location must be unknown since the first debug location is used
819  // to determine the end of the prologue.
820  DebugLoc DL;
821 
822  if (ShouldSignReturnAddress(MF)) {
823  if (ShouldSignWithAKey(MF))
824  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
825  .setMIFlag(MachineInstr::FrameSetup);
826  else {
827  BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
828  .setMIFlag(MachineInstr::FrameSetup);
829  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
830  .setMIFlag(MachineInstr::FrameSetup);
831  }
832 
833  unsigned CFIIndex =
835  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
836  .addCFIIndex(CFIIndex)
838  }
839 
840  // All calls are tail calls in GHC calling conv, and functions have no
841  // prologue/epilogue.
843  return;
844 
845  // getStackSize() includes all the locals in its size calculation. We don't
846  // include these locals when computing the stack size of a funclet, as they
847  // are allocated in the parent's stack frame and accessed via the frame
848  // pointer from the funclet. We only save the callee saved registers in the
849  // funclet, which are really the callee saved registers of the parent
850  // function, including the funclet.
851  int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
852  : (int)MFI.getStackSize();
853  if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
854  assert(!HasFP && "unexpected function without stack frame but with FP");
855  // All of the stack allocation is for locals.
856  AFI->setLocalStackSize(NumBytes);
857  if (!NumBytes)
858  return;
859  // REDZONE: If the stack size is less than 128 bytes, we don't need
860  // to actually allocate.
861  if (canUseRedZone(MF)) {
862  AFI->setHasRedZone(true);
863  ++NumRedZoneFunctions;
864  } else {
865  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
866  MachineInstr::FrameSetup, false, NeedsWinCFI);
867  if (!NeedsWinCFI) {
868  // Label used to tie together the PROLOG_LABEL and the MachineMoves.
869  MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
870  // Encode the stack size of the leaf function.
871  unsigned CFIIndex = MF.addFrameInst(
872  MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
873  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
874  .addCFIIndex(CFIIndex)
876  }
877  }
878 
879  if (NeedsWinCFI)
880  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
881  .setMIFlag(MachineInstr::FrameSetup);
882 
883  return;
884  }
885 
886  bool IsWin64 =
888  // Var args are accounted for in the containing function, so don't
889  // include them for funclets.
890  unsigned FixedObject = (IsWin64 && !IsFunclet) ?
891  alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
892 
893  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
894  // All of the remaining stack allocations are for locals.
895  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
896  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
897  if (CombineSPBump) {
898  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
899  MachineInstr::FrameSetup, false, NeedsWinCFI);
900  NumBytes = 0;
901  } else if (PrologueSaveSize != 0) {
903  MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI);
904  NumBytes -= PrologueSaveSize;
905  }
906  assert(NumBytes >= 0 && "Negative stack allocation size!?");
907 
908  // Move past the saves of the callee-saved registers, fixing up the offsets
909  // and pre-inc if we decided to combine the callee-save and local stack
910  // pointer bump above.
911  MachineBasicBlock::iterator End = MBB.end();
912  while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
913  if (CombineSPBump)
915  NeedsWinCFI);
916  ++MBBI;
917  }
918 
919  // The code below is not applicable to funclets. We have emitted all the SEH
920  // opcodes that we needed to emit. The FP and BP belong to the containing
921  // function.
922  if (IsFunclet) {
923  if (NeedsWinCFI)
924  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
925  .setMIFlag(MachineInstr::FrameSetup);
926 
927  // SEH funclets are passed the frame pointer in X1. If the parent
928  // function uses the base register, then the base register is used
929  // directly, and is not retrieved from X1.
930  if (F.hasPersonalityFn()) {
932  if (isAsynchronousEHPersonality(Per)) {
933  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
934  .addReg(AArch64::X1).setMIFlag(MachineInstr::FrameSetup);
935  MBB.addLiveIn(AArch64::X1);
936  }
937  }
938 
939  return;
940  }
941 
942  if (HasFP) {
943  // Only set up FP if we actually need to. Frame pointer is fp =
944  // sp - fixedobject - 16.
945  int FPOffset = AFI->getCalleeSavedStackSize() - 16;
946  if (CombineSPBump)
947  FPOffset += AFI->getLocalStackSize();
948 
949  // Issue sub fp, sp, FPOffset or
950  // mov fp,sp when FPOffset is zero.
951  // Note: All stores of callee-saved registers are marked as "FrameSetup".
952  // This code marks the instruction(s) that set the FP also.
953  emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
954  MachineInstr::FrameSetup, false, NeedsWinCFI);
955  }
956 
957  if (windowsRequiresStackProbe(MF, NumBytes)) {
958  uint32_t NumWords = NumBytes >> 4;
959  if (NeedsWinCFI) {
960  // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
961  // exceed this amount. We need to move at most 2^24 - 1 into x15.
962  // This is at most two instructions, MOVZ follwed by MOVK.
963  // TODO: Fix to use multiple stack alloc unwind codes for stacks
964  // exceeding 256MB in size.
965  if (NumBytes >= (1 << 28))
966  report_fatal_error("Stack size cannot exceed 256MB for stack "
967  "unwinding purposes");
968 
969  uint32_t LowNumWords = NumWords & 0xFFFF;
970  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
971  .addImm(LowNumWords)
974  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
975  .setMIFlag(MachineInstr::FrameSetup);
976  if ((NumWords & 0xFFFF0000) != 0) {
977  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
978  .addReg(AArch64::X15)
979  .addImm((NumWords & 0xFFFF0000) >> 16) // High half
982  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
983  .setMIFlag(MachineInstr::FrameSetup);
984  }
985  } else {
986  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
987  .addImm(NumWords)
989  }
990 
991  switch (MF.getTarget().getCodeModel()) {
992  case CodeModel::Tiny:
993  case CodeModel::Small:
994  case CodeModel::Medium:
995  case CodeModel::Kernel:
996  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
997  .addExternalSymbol("__chkstk")
998  .addReg(AArch64::X15, RegState::Implicit)
1003  if (NeedsWinCFI)
1004  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1005  .setMIFlag(MachineInstr::FrameSetup);
1006  break;
1007  case CodeModel::Large:
1008  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1009  .addReg(AArch64::X16, RegState::Define)
1010  .addExternalSymbol("__chkstk")
1011  .addExternalSymbol("__chkstk")
1013  if (NeedsWinCFI)
1014  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1015  .setMIFlag(MachineInstr::FrameSetup);
1016 
1017  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
1018  .addReg(AArch64::X16, RegState::Kill)
1019  .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1024  if (NeedsWinCFI)
1025  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1026  .setMIFlag(MachineInstr::FrameSetup);
1027  break;
1028  }
1029 
1030  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1031  .addReg(AArch64::SP, RegState::Kill)
1032  .addReg(AArch64::X15, RegState::Kill)
1035  if (NeedsWinCFI)
1036  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1037  .addImm(NumBytes)
1039  NumBytes = 0;
1040  }
1041 
1042  // Allocate space for the rest of the frame.
1043  if (NumBytes) {
1044  const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
1045  unsigned scratchSPReg = AArch64::SP;
1046 
1047  if (NeedsRealignment) {
1048  scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1049  assert(scratchSPReg != AArch64::NoRegister);
1050  }
1051 
1052  // If we're a leaf function, try using the red zone.
1053  if (!canUseRedZone(MF))
1054  // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1055  // the correct value here, as NumBytes also includes padding bytes,
1056  // which shouldn't be counted here.
1057  emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
1058  MachineInstr::FrameSetup, false, NeedsWinCFI);
1059 
1060  if (NeedsRealignment) {
1061  const unsigned Alignment = MFI.getMaxAlignment();
1062  const unsigned NrBitsToZero = countTrailingZeros(Alignment);
1063  assert(NrBitsToZero > 1);
1064  assert(scratchSPReg != AArch64::SP);
1065 
1066  // SUB X9, SP, NumBytes
1067  // -- X9 is temporary register, so shouldn't contain any live data here,
1068  // -- free to use. This is already produced by emitFrameOffset above.
1069  // AND SP, X9, 0b11111...0000
1070  // The logical immediates have a non-trivial encoding. The following
1071  // formula computes the encoded immediate with all ones but
1072  // NrBitsToZero zero bits as least significant bits.
1073  uint32_t andMaskEncoded = (1 << 12) // = N
1074  | ((64 - NrBitsToZero) << 6) // immr
1075  | ((64 - NrBitsToZero - 1) << 0); // imms
1076 
1077  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1078  .addReg(scratchSPReg, RegState::Kill)
1079  .addImm(andMaskEncoded);
1080  AFI->setStackRealigned(true);
1081  if (NeedsWinCFI)
1082  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1083  .addImm(NumBytes & andMaskEncoded)
1085  }
1086  }
1087 
1088  // If we need a base pointer, set it up here. It's whatever the value of the
1089  // stack pointer is at this point. Any variable size objects will be allocated
1090  // after this, so we can still use the base pointer to reference locals.
1091  //
1092  // FIXME: Clarify FrameSetup flags here.
1093  // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1094  // needed.
1095  if (RegInfo->hasBasePointer(MF)) {
1096  TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1097  false);
1098  if (NeedsWinCFI)
1099  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1100  .setMIFlag(MachineInstr::FrameSetup);
1101  }
1102 
1103  // The very last FrameSetup instruction indicates the end of prologue. Emit a
1104  // SEH opcode indicating the prologue end.
1105  if (NeedsWinCFI)
1106  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1107  .setMIFlag(MachineInstr::FrameSetup);
1108 
1109  if (needsFrameMoves) {
1110  const DataLayout &TD = MF.getDataLayout();
1111  const int StackGrowth = -TD.getPointerSize(0);
1112  unsigned FramePtr = RegInfo->getFrameRegister(MF);
1113  // An example of the prologue:
1114  //
1115  // .globl __foo
1116  // .align 2
1117  // __foo:
1118  // Ltmp0:
1119  // .cfi_startproc
1120  // .cfi_personality 155, ___gxx_personality_v0
1121  // Leh_func_begin:
1122  // .cfi_lsda 16, Lexception33
1123  //
1124  // stp xa,bx, [sp, -#offset]!
1125  // ...
1126  // stp x28, x27, [sp, #offset-32]
1127  // stp fp, lr, [sp, #offset-16]
1128  // add fp, sp, #offset - 16
1129  // sub sp, sp, #1360
1130  //
1131  // The Stack:
1132  // +-------------------------------------------+
1133  // 10000 | ........ | ........ | ........ | ........ |
1134  // 10004 | ........ | ........ | ........ | ........ |
1135  // +-------------------------------------------+
1136  // 10008 | ........ | ........ | ........ | ........ |
1137  // 1000c | ........ | ........ | ........ | ........ |
1138  // +===========================================+
1139  // 10010 | X28 Register |
1140  // 10014 | X28 Register |
1141  // +-------------------------------------------+
1142  // 10018 | X27 Register |
1143  // 1001c | X27 Register |
1144  // +===========================================+
1145  // 10020 | Frame Pointer |
1146  // 10024 | Frame Pointer |
1147  // +-------------------------------------------+
1148  // 10028 | Link Register |
1149  // 1002c | Link Register |
1150  // +===========================================+
1151  // 10030 | ........ | ........ | ........ | ........ |
1152  // 10034 | ........ | ........ | ........ | ........ |
1153  // +-------------------------------------------+
1154  // 10038 | ........ | ........ | ........ | ........ |
1155  // 1003c | ........ | ........ | ........ | ........ |
1156  // +-------------------------------------------+
1157  //
1158  // [sp] = 10030 :: >>initial value<<
1159  // sp = 10020 :: stp fp, lr, [sp, #-16]!
1160  // fp = sp == 10020 :: mov fp, sp
1161  // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1162  // sp == 10010 :: >>final value<<
1163  //
1164  // The frame pointer (w29) points to address 10020. If we use an offset of
1165  // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1166  // for w27, and -32 for w28:
1167  //
1168  // Ltmp1:
1169  // .cfi_def_cfa w29, 16
1170  // Ltmp2:
1171  // .cfi_offset w30, -8
1172  // Ltmp3:
1173  // .cfi_offset w29, -16
1174  // Ltmp4:
1175  // .cfi_offset w27, -24
1176  // Ltmp5:
1177  // .cfi_offset w28, -32
1178 
1179  if (HasFP) {
1180  // Define the current CFA rule to use the provided FP.
1181  unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1182  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
1183  nullptr, Reg, 2 * StackGrowth - FixedObject));
1184  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1185  .addCFIIndex(CFIIndex)
1187  } else {
1188  // Encode the stack size of the leaf function.
1189  unsigned CFIIndex = MF.addFrameInst(
1191  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1192  .addCFIIndex(CFIIndex)
1194  }
1195 
1196  // Now emit the moves for whatever callee saved regs we have (including FP,
1197  // LR if those are saved).
1198  emitCalleeSavedFrameMoves(MBB, MBBI);
1199  }
1200 }
1201 
1203  MachineBasicBlock &MBB) {
1204  if (!ShouldSignReturnAddress(MF))
1205  return;
1206  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1207  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1208 
1210  DebugLoc DL;
1211  if (MBBI != MBB.end())
1212  DL = MBBI->getDebugLoc();
1213 
1214  // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1215  // this instruction can safely used for any v8a architecture.
1216  // From v8.3a onwards there are optimised authenticate LR and return
1217  // instructions, namely RETA{A,B}, that can be used instead.
1218  if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
1219  MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1220  BuildMI(MBB, MBBI, DL,
1221  TII->get(ShouldSignWithAKey(MF) ? AArch64::RETAA : AArch64::RETAB))
1222  .copyImplicitOps(*MBBI);
1223  MBB.erase(MBBI);
1224  } else {
1225  BuildMI(
1226  MBB, MBBI, DL,
1227  TII->get(ShouldSignWithAKey(MF) ? AArch64::AUTIASP : AArch64::AUTIBSP))
1228  .setMIFlag(MachineInstr::FrameDestroy);
1229  }
1230 }
1231 
1232 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1233  switch (MI.getOpcode()) {
1234  default:
1235  return false;
1236  case AArch64::CATCHRET:
1237  case AArch64::CLEANUPRET:
1238  return true;
1239  }
1240 }
1241 
1243  MachineBasicBlock &MBB) const {
1245  MachineFrameInfo &MFI = MF.getFrameInfo();
1246  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1247  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1248  DebugLoc DL;
1249  bool IsTailCallReturn = false;
1250  bool NeedsWinCFI = needsWinCFI(MF);
1251  bool IsFunclet = false;
1252 
1253  if (MBB.end() != MBBI) {
1254  DL = MBBI->getDebugLoc();
1255  unsigned RetOpcode = MBBI->getOpcode();
1256  IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
1257  RetOpcode == AArch64::TCRETURNri ||
1258  RetOpcode == AArch64::TCRETURNriBTI;
1259  IsFunclet = isFuncletReturnInstr(*MBBI);
1260  }
1261 
1262  int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
1263  : MFI.getStackSize();
1265 
1266  // All calls are tail calls in GHC calling conv, and functions have no
1267  // prologue/epilogue.
1269  return;
1270 
1271  // Initial and residual are named for consistency with the prologue. Note that
1272  // in the epilogue, the residual adjustment is executed first.
1273  uint64_t ArgumentPopSize = 0;
1274  if (IsTailCallReturn) {
1275  MachineOperand &StackAdjust = MBBI->getOperand(1);
1276 
1277  // For a tail-call in a callee-pops-arguments environment, some or all of
1278  // the stack may actually be in use for the call's arguments, this is
1279  // calculated during LowerCall and consumed here...
1280  ArgumentPopSize = StackAdjust.getImm();
1281  } else {
1282  // ... otherwise the amount to pop is *all* of the argument space,
1283  // conveniently stored in the MachineFunctionInfo by
1284  // LowerFormalArguments. This will, of course, be zero for the C calling
1285  // convention.
1286  ArgumentPopSize = AFI->getArgumentStackToRestore();
1287  }
1288 
1289  // The stack frame should be like below,
1290  //
1291  // ---------------------- ---
1292  // | | |
1293  // | BytesInStackArgArea| CalleeArgStackSize
1294  // | (NumReusableBytes) | (of tail call)
1295  // | | ---
1296  // | | |
1297  // ---------------------| --- |
1298  // | | | |
1299  // | CalleeSavedReg | | |
1300  // | (CalleeSavedStackSize)| | |
1301  // | | | |
1302  // ---------------------| | NumBytes
1303  // | | StackSize (StackAdjustUp)
1304  // | LocalStackSize | | |
1305  // | (covering callee | | |
1306  // | args) | | |
1307  // | | | |
1308  // ---------------------- --- ---
1309  //
1310  // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1311  // = StackSize + ArgumentPopSize
1312  //
1313  // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1314  // it as the 2nd argument of AArch64ISD::TC_RETURN.
1315 
1316  auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1317 
1318  bool IsWin64 =
1319  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1320  // Var args are accounted for in the containing function, so don't
1321  // include them for funclets.
1322  unsigned FixedObject =
1323  (IsWin64 && !IsFunclet) ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1324 
1325  uint64_t AfterCSRPopSize = ArgumentPopSize;
1326  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1327  // We cannot rely on the local stack size set in emitPrologue if the function
1328  // has funclets, as funclets have different local stack size requirements, and
1329  // the current value set in emitPrologue may be that of the containing
1330  // function.
1331  if (MF.hasEHFunclets())
1332  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1333  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1334  // Assume we can't combine the last pop with the sp restore.
1335 
1336  if (!CombineSPBump && PrologueSaveSize != 0) {
1337  MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
1339  Pop = std::prev(Pop);
1340  // Converting the last ldp to a post-index ldp is valid only if the last
1341  // ldp's offset is 0.
1342  const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1343  // If the offset is 0, convert it to a post-index ldp.
1344  if (OffsetOp.getImm() == 0)
1346  MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, false);
1347  else {
1348  // If not, make sure to emit an add after the last ldp.
1349  // We're doing this by transfering the size to be restored from the
1350  // adjustment *before* the CSR pops to the adjustment *after* the CSR
1351  // pops.
1352  AfterCSRPopSize += PrologueSaveSize;
1353  }
1354  }
1355 
1356  // Move past the restores of the callee-saved registers.
1357  // If we plan on combining the sp bump of the local stack size and the callee
1358  // save stack size, we might need to adjust the CSR save and restore offsets.
1360  MachineBasicBlock::iterator Begin = MBB.begin();
1361  while (LastPopI != Begin) {
1362  --LastPopI;
1363  if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
1364  ++LastPopI;
1365  break;
1366  } else if (CombineSPBump)
1368  NeedsWinCFI);
1369  }
1370 
1371  if (NeedsWinCFI)
1372  BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1373  .setMIFlag(MachineInstr::FrameDestroy);
1374 
1375  // If there is a single SP update, insert it before the ret and we're done.
1376  if (CombineSPBump) {
1377  emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1378  NumBytes + AfterCSRPopSize, TII, MachineInstr::FrameDestroy,
1379  false, NeedsWinCFI);
1380  if (NeedsWinCFI)
1381  BuildMI(MBB, MBB.getFirstTerminator(), DL,
1382  TII->get(AArch64::SEH_EpilogEnd))
1383  .setMIFlag(MachineInstr::FrameDestroy);
1384  return;
1385  }
1386 
1387  NumBytes -= PrologueSaveSize;
1388  assert(NumBytes >= 0 && "Negative stack allocation size!?");
1389 
1390  if (!hasFP(MF)) {
1391  bool RedZone = canUseRedZone(MF);
1392  // If this was a redzone leaf function, we don't need to restore the
1393  // stack pointer (but we may need to pop stack args for fastcc).
1394  if (RedZone && AfterCSRPopSize == 0)
1395  return;
1396 
1397  bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1398  int StackRestoreBytes = RedZone ? 0 : NumBytes;
1399  if (NoCalleeSaveRestore)
1400  StackRestoreBytes += AfterCSRPopSize;
1401 
1402  // If we were able to combine the local stack pop with the argument pop,
1403  // then we're done.
1404  bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1405 
1406  // If we're done after this, make sure to help the load store optimizer.
1407  if (Done)
1408  adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1409 
1410  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1411  StackRestoreBytes, TII, MachineInstr::FrameDestroy, false,
1412  NeedsWinCFI);
1413  if (Done) {
1414  if (NeedsWinCFI)
1415  BuildMI(MBB, MBB.getFirstTerminator(), DL,
1416  TII->get(AArch64::SEH_EpilogEnd))
1417  .setMIFlag(MachineInstr::FrameDestroy);
1418  return;
1419  }
1420 
1421  NumBytes = 0;
1422  }
1423 
1424  // Restore the original stack pointer.
1425  // FIXME: Rather than doing the math here, we should instead just use
1426  // non-post-indexed loads for the restores if we aren't actually going to
1427  // be able to save any instructions.
1428  if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned()))
1429  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1430  -AFI->getCalleeSavedStackSize() + 16, TII,
1431  MachineInstr::FrameDestroy, false, NeedsWinCFI);
1432  else if (NumBytes)
1433  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
1434  MachineInstr::FrameDestroy, false, NeedsWinCFI);
1435 
1436  // This must be placed after the callee-save restore code because that code
1437  // assumes the SP is at the same location as it was after the callee-save save
1438  // code in the prologue.
1439  if (AfterCSRPopSize) {
1440  // Find an insertion point for the first ldp so that it goes before the
1441  // shadow call stack epilog instruction. This ensures that the restore of
1442  // lr from x18 is placed after the restore from sp.
1443  auto FirstSPPopI = MBB.getFirstTerminator();
1444  while (FirstSPPopI != Begin) {
1445  auto Prev = std::prev(FirstSPPopI);
1446  if (Prev->getOpcode() != AArch64::LDRXpre ||
1447  Prev->getOperand(0).getReg() == AArch64::SP)
1448  break;
1449  FirstSPPopI = Prev;
1450  }
1451 
1452  adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1453 
1454  emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1455  AfterCSRPopSize, TII, MachineInstr::FrameDestroy, false,
1456  NeedsWinCFI);
1457  }
1458  if (NeedsWinCFI)
1459  BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1460  .setMIFlag(MachineInstr::FrameDestroy);
1461 }
1462 
1463 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1464 /// debug info. It's the same as what we use for resolving the code-gen
1465 /// references for now. FIXME: This can go wrong when references are
1466 /// SP-relative and simple call frames aren't used.
1468  int FI,
1469  unsigned &FrameReg) const {
1470  return resolveFrameIndexReference(MF, FI, FrameReg);
1471 }
1472 
1474  int FI, unsigned &FrameReg,
1475  bool PreferFP) const {
1476  const MachineFrameInfo &MFI = MF.getFrameInfo();
1477  const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
1478  MF.getSubtarget().getRegisterInfo());
1479  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1480  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1481  bool IsWin64 =
1482  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1483  unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1484  int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16;
1485  int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
1486  bool isFixed = MFI.isFixedObjectIndex(FI);
1487  bool isCSR = !isFixed && MFI.getObjectOffset(FI) >=
1488  -((int)AFI->getCalleeSavedStackSize());
1489 
1490  // Use frame pointer to reference fixed objects. Use it for locals if
1491  // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1492  // reliable as a base). Make sure useFPForScavengingIndex() does the
1493  // right thing for the emergency spill slot.
1494  bool UseFP = false;
1495  if (AFI->hasStackFrame()) {
1496  // Note: Keeping the following as multiple 'if' statements rather than
1497  // merging to a single expression for readability.
1498  //
1499  // Argument access should always use the FP.
1500  if (isFixed) {
1501  UseFP = hasFP(MF);
1502  } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1503  // References to the CSR area must use FP if we're re-aligning the stack
1504  // since the dynamically-sized alignment padding is between the SP/BP and
1505  // the CSR area.
1506  assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1507  UseFP = true;
1508  } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1509  // If the FPOffset is negative, we have to keep in mind that the
1510  // available offset range for negative offsets is smaller than for
1511  // positive ones. If an offset is
1512  // available via the FP and the SP, use whichever is closest.
1513  bool FPOffsetFits = FPOffset >= -256;
1514  PreferFP |= Offset > -FPOffset;
1515 
1516  if (MFI.hasVarSizedObjects()) {
1517  // If we have variable sized objects, we can use either FP or BP, as the
1518  // SP offset is unknown. We can use the base pointer if we have one and
1519  // FP is not preferred. If not, we're stuck with using FP.
1520  bool CanUseBP = RegInfo->hasBasePointer(MF);
1521  if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1522  UseFP = PreferFP;
1523  else if (!CanUseBP) // Can't use BP. Forced to use FP.
1524  UseFP = true;
1525  // else we can use BP and FP, but the offset from FP won't fit.
1526  // That will make us scavenge registers which we can probably avoid by
1527  // using BP. If it won't fit for BP either, we'll scavenge anyway.
1528  } else if (FPOffset >= 0) {
1529  // Use SP or FP, whichever gives us the best chance of the offset
1530  // being in range for direct access. If the FPOffset is positive,
1531  // that'll always be best, as the SP will be even further away.
1532  UseFP = true;
1533  } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1534  // Funclets access the locals contained in the parent's stack frame
1535  // via the frame pointer, so we have to use the FP in the parent
1536  // function.
1537  assert(
1538  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
1539  "Funclets should only be present on Win64");
1540  UseFP = true;
1541  } else {
1542  // We have the choice between FP and (SP or BP).
1543  if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1544  UseFP = true;
1545  }
1546  }
1547  }
1548 
1549  assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1550  "In the presence of dynamic stack pointer realignment, "
1551  "non-argument/CSR objects cannot be accessed through the frame pointer");
1552 
1553  if (UseFP) {
1554  FrameReg = RegInfo->getFrameRegister(MF);
1555  return FPOffset;
1556  }
1557 
1558  // Use the base pointer if we have one.
1559  if (RegInfo->hasBasePointer(MF))
1560  FrameReg = RegInfo->getBaseRegister();
1561  else {
1562  assert(!MFI.hasVarSizedObjects() &&
1563  "Can't use SP when we have var sized objects.");
1564  FrameReg = AArch64::SP;
1565  // If we're using the red zone for this function, the SP won't actually
1566  // be adjusted, so the offsets will be negative. They're also all
1567  // within range of the signed 9-bit immediate instructions.
1568  if (canUseRedZone(MF))
1569  Offset -= AFI->getLocalStackSize();
1570  }
1571 
1572  return Offset;
1573 }
1574 
1575 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1576  // Do not set a kill flag on values that are also marked as live-in. This
1577  // happens with the @llvm-returnaddress intrinsic and with arguments passed in
1578  // callee saved registers.
1579  // Omitting the kill flags is conservatively correct even if the live-in
1580  // is not used after all.
1581  bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1582  return getKillRegState(!IsLiveIn);
1583 }
1584 
1586  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1588  return Subtarget.isTargetMachO() &&
1589  !(Subtarget.getTargetLowering()->supportSwiftError() &&
1590  Attrs.hasAttrSomewhere(Attribute::SwiftError));
1591 }
1592 
1593 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
1594  bool NeedsWinCFI) {
1595  // If we are generating register pairs for a Windows function that requires
1596  // EH support, then pair consecutive registers only. There are no unwind
1597  // opcodes for saves/restores of non-consectuve register pairs.
1598  // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x.
1599  // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1600 
1601  // TODO: LR can be paired with any register. We don't support this yet in
1602  // the MCLayer. We need to add support for the save_lrpair unwind code.
1603  if (!NeedsWinCFI)
1604  return false;
1605  if (Reg2 == Reg1 + 1)
1606  return false;
1607  return true;
1608 }
1609 
1610 namespace {
1611 
1612 struct RegPairInfo {
1613  unsigned Reg1 = AArch64::NoRegister;
1614  unsigned Reg2 = AArch64::NoRegister;
1615  int FrameIdx;
1616  int Offset;
1617  enum RegType { GPR, FPR64, FPR128 } Type;
1618 
1619  RegPairInfo() = default;
1620 
1621  bool isPaired() const { return Reg2 != AArch64::NoRegister; }
1622 };
1623 
1624 } // end anonymous namespace
1625 
1627  MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
1629  bool &NeedShadowCallStackProlog) {
1630 
1631  if (CSI.empty())
1632  return;
1633 
1634  bool NeedsWinCFI = needsWinCFI(MF);
1636  MachineFrameInfo &MFI = MF.getFrameInfo();
1638  unsigned Count = CSI.size();
1639  (void)CC;
1640  // MachO's compact unwind format relies on all registers being stored in
1641  // pairs.
1643  CC == CallingConv::PreserveMost ||
1644  (Count & 1) == 0) &&
1645  "Odd number of callee-saved regs to spill!");
1646  int Offset = AFI->getCalleeSavedStackSize();
1647  // On Linux, we will have either one or zero non-paired register. On Windows
1648  // with CFI, we can have multiple unpaired registers in order to utilize the
1649  // available unwind codes. This flag assures that the alignment fixup is done
1650  // only once, as intened.
1651  bool FixupDone = false;
1652  for (unsigned i = 0; i < Count; ++i) {
1653  RegPairInfo RPI;
1654  RPI.Reg1 = CSI[i].getReg();
1655 
1656  if (AArch64::GPR64RegClass.contains(RPI.Reg1))
1657  RPI.Type = RegPairInfo::GPR;
1658  else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
1659  RPI.Type = RegPairInfo::FPR64;
1660  else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
1661  RPI.Type = RegPairInfo::FPR128;
1662  else
1663  llvm_unreachable("Unsupported register class.");
1664 
1665  // Add the next reg to the pair if it is in the same register class.
1666  if (i + 1 < Count) {
1667  unsigned NextReg = CSI[i + 1].getReg();
1668  switch (RPI.Type) {
1669  case RegPairInfo::GPR:
1670  if (AArch64::GPR64RegClass.contains(NextReg) &&
1671  !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
1672  RPI.Reg2 = NextReg;
1673  break;
1674  case RegPairInfo::FPR64:
1675  if (AArch64::FPR64RegClass.contains(NextReg) &&
1676  !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
1677  RPI.Reg2 = NextReg;
1678  break;
1679  case RegPairInfo::FPR128:
1680  if (AArch64::FPR128RegClass.contains(NextReg))
1681  RPI.Reg2 = NextReg;
1682  break;
1683  }
1684  }
1685 
1686  // If either of the registers to be saved is the lr register, it means that
1687  // we also need to save lr in the shadow call stack.
1688  if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
1689  MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
1691  report_fatal_error("Must reserve x18 to use shadow call stack");
1692  NeedShadowCallStackProlog = true;
1693  }
1694 
1695  // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
1696  // list to come in sorted by frame index so that we can issue the store
1697  // pair instructions directly. Assert if we see anything otherwise.
1698  //
1699  // The order of the registers in the list is controlled by
1700  // getCalleeSavedRegs(), so they will always be in-order, as well.
1701  assert((!RPI.isPaired() ||
1702  (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
1703  "Out of order callee saved regs!");
1704 
1705  // MachO's compact unwind format relies on all registers being stored in
1706  // adjacent register pairs.
1708  CC == CallingConv::PreserveMost ||
1709  (RPI.isPaired() &&
1710  ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
1711  RPI.Reg1 + 1 == RPI.Reg2))) &&
1712  "Callee-save registers not saved as adjacent register pair!");
1713 
1714  RPI.FrameIdx = CSI[i].getFrameIdx();
1715 
1716  int Scale = RPI.Type == RegPairInfo::FPR128 ? 16 : 8;
1717  Offset -= RPI.isPaired() ? 2 * Scale : Scale;
1718 
1719  // Round up size of non-pair to pair size if we need to pad the
1720  // callee-save area to ensure 16-byte alignment.
1721  if (AFI->hasCalleeSaveStackFreeSpace() && !FixupDone &&
1722  RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired()) {
1723  FixupDone = true;
1724  Offset -= 8;
1725  assert(Offset % 16 == 0);
1726  assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
1727  MFI.setObjectAlignment(RPI.FrameIdx, 16);
1728  }
1729 
1730  assert(Offset % Scale == 0);
1731  RPI.Offset = Offset / Scale;
1732  assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
1733  "Offset out of bounds for LDP/STP immediate");
1734 
1735  RegPairs.push_back(RPI);
1736  if (RPI.isPaired())
1737  ++i;
1738  }
1739 }
1740 
1743  const std::vector<CalleeSavedInfo> &CSI,
1744  const TargetRegisterInfo *TRI) const {
1745  MachineFunction &MF = *MBB.getParent();
1746  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1747  bool NeedsWinCFI = needsWinCFI(MF);
1748  DebugLoc DL;
1749  SmallVector<RegPairInfo, 8> RegPairs;
1750 
1751  bool NeedShadowCallStackProlog = false;
1752  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1753  NeedShadowCallStackProlog);
1754  const MachineRegisterInfo &MRI = MF.getRegInfo();
1755 
1756  if (NeedShadowCallStackProlog) {
1757  // Shadow call stack prolog: str x30, [x18], #8
1758  BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
1759  .addReg(AArch64::X18, RegState::Define)
1760  .addReg(AArch64::LR)
1761  .addReg(AArch64::X18)
1762  .addImm(8)
1764 
1765  if (NeedsWinCFI)
1766  BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
1767  .setMIFlag(MachineInstr::FrameSetup);
1768 
1769  if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
1770  // Emit a CFI instruction that causes 8 to be subtracted from the value of
1771  // x18 when unwinding past this frame.
1772  static const char CFIInst[] = {
1773  dwarf::DW_CFA_val_expression,
1774  18, // register
1775  2, // length
1776  static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
1777  static_cast<char>(-8) & 0x7f, // addend (sleb128)
1778  };
1779  unsigned CFIIndex =
1780  MF.addFrameInst(MCCFIInstruction::createEscape(nullptr, CFIInst));
1781  BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
1782  .addCFIIndex(CFIIndex)
1784  }
1785 
1786  // This instruction also makes x18 live-in to the entry block.
1787  MBB.addLiveIn(AArch64::X18);
1788  }
1789 
1790  for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
1791  ++RPII) {
1792  RegPairInfo RPI = *RPII;
1793  unsigned Reg1 = RPI.Reg1;
1794  unsigned Reg2 = RPI.Reg2;
1795  unsigned StrOpc;
1796 
1797  // Issue sequence of spills for cs regs. The first spill may be converted
1798  // to a pre-decrement store later by emitPrologue if the callee-save stack
1799  // area allocation can't be combined with the local stack area allocation.
1800  // For example:
1801  // stp x22, x21, [sp, #0] // addImm(+0)
1802  // stp x20, x19, [sp, #16] // addImm(+2)
1803  // stp fp, lr, [sp, #32] // addImm(+4)
1804  // Rationale: This sequence saves uop updates compared to a sequence of
1805  // pre-increment spills like stp xi,xj,[sp,#-16]!
1806  // Note: Similar rationale and sequence for restores in epilog.
1807  unsigned Size, Align;
1808  switch (RPI.Type) {
1809  case RegPairInfo::GPR:
1810  StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
1811  Size = 8;
1812  Align = 8;
1813  break;
1814  case RegPairInfo::FPR64:
1815  StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
1816  Size = 8;
1817  Align = 8;
1818  break;
1819  case RegPairInfo::FPR128:
1820  StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
1821  Size = 16;
1822  Align = 16;
1823  break;
1824  }
1825  LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
1826  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1827  dbgs() << ") -> fi#(" << RPI.FrameIdx;
1828  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1829  dbgs() << ")\n");
1830 
1831  assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
1832  "Windows unwdinding requires a consecutive (FP,LR) pair");
1833  // Windows unwind codes require consecutive registers if registers are
1834  // paired. Make the switch here, so that the code below will save (x,x+1)
1835  // and not (x+1,x).
1836  unsigned FrameIdxReg1 = RPI.FrameIdx;
1837  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
1838  if (NeedsWinCFI && RPI.isPaired()) {
1839  std::swap(Reg1, Reg2);
1840  std::swap(FrameIdxReg1, FrameIdxReg2);
1841  }
1842  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
1843  if (!MRI.isReserved(Reg1))
1844  MBB.addLiveIn(Reg1);
1845  if (RPI.isPaired()) {
1846  if (!MRI.isReserved(Reg2))
1847  MBB.addLiveIn(Reg2);
1848  MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
1850  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
1851  MachineMemOperand::MOStore, Size, Align));
1852  }
1853  MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
1854  .addReg(AArch64::SP)
1855  .addImm(RPI.Offset) // [sp, #offset*scale],
1856  // where factor*scale is implicit
1859  MachinePointerInfo::getFixedStack(MF,FrameIdxReg1),
1860  MachineMemOperand::MOStore, Size, Align));
1861  if (NeedsWinCFI)
1863 
1864  }
1865  return true;
1866 }
1867 
1870  std::vector<CalleeSavedInfo> &CSI,
1871  const TargetRegisterInfo *TRI) const {
1872  MachineFunction &MF = *MBB.getParent();
1873  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1874  DebugLoc DL;
1875  SmallVector<RegPairInfo, 8> RegPairs;
1876  bool NeedsWinCFI = needsWinCFI(MF);
1877 
1878  if (MI != MBB.end())
1879  DL = MI->getDebugLoc();
1880 
1881  bool NeedShadowCallStackProlog = false;
1882  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1883  NeedShadowCallStackProlog);
1884 
1885  auto EmitMI = [&](const RegPairInfo &RPI) {
1886  unsigned Reg1 = RPI.Reg1;
1887  unsigned Reg2 = RPI.Reg2;
1888 
1889  // Issue sequence of restores for cs regs. The last restore may be converted
1890  // to a post-increment load later by emitEpilogue if the callee-save stack
1891  // area allocation can't be combined with the local stack area allocation.
1892  // For example:
1893  // ldp fp, lr, [sp, #32] // addImm(+4)
1894  // ldp x20, x19, [sp, #16] // addImm(+2)
1895  // ldp x22, x21, [sp, #0] // addImm(+0)
1896  // Note: see comment in spillCalleeSavedRegisters()
1897  unsigned LdrOpc;
1898  unsigned Size, Align;
1899  switch (RPI.Type) {
1900  case RegPairInfo::GPR:
1901  LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
1902  Size = 8;
1903  Align = 8;
1904  break;
1905  case RegPairInfo::FPR64:
1906  LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
1907  Size = 8;
1908  Align = 8;
1909  break;
1910  case RegPairInfo::FPR128:
1911  LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
1912  Size = 16;
1913  Align = 16;
1914  break;
1915  }
1916  LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
1917  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
1918  dbgs() << ") -> fi#(" << RPI.FrameIdx;
1919  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
1920  dbgs() << ")\n");
1921 
1922  // Windows unwind codes require consecutive registers if registers are
1923  // paired. Make the switch here, so that the code below will save (x,x+1)
1924  // and not (x+1,x).
1925  unsigned FrameIdxReg1 = RPI.FrameIdx;
1926  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
1927  if (NeedsWinCFI && RPI.isPaired()) {
1928  std::swap(Reg1, Reg2);
1929  std::swap(FrameIdxReg1, FrameIdxReg2);
1930  }
1931  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
1932  if (RPI.isPaired()) {
1933  MIB.addReg(Reg2, getDefRegState(true));
1935  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
1936  MachineMemOperand::MOLoad, Size, Align));
1937  }
1938  MIB.addReg(Reg1, getDefRegState(true))
1939  .addReg(AArch64::SP)
1940  .addImm(RPI.Offset) // [sp, #offset*scale]
1941  // where factor*scale is implicit
1944  MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
1945  MachineMemOperand::MOLoad, Size, Align));
1946  if (NeedsWinCFI)
1948  };
1950  for (const RegPairInfo &RPI : reverse(RegPairs))
1951  EmitMI(RPI);
1952  else
1953  for (const RegPairInfo &RPI : RegPairs)
1954  EmitMI(RPI);
1955 
1956  if (NeedShadowCallStackProlog) {
1957  // Shadow call stack epilog: ldr x30, [x18, #-8]!
1958  BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
1959  .addReg(AArch64::X18, RegState::Define)
1960  .addReg(AArch64::LR, RegState::Define)
1961  .addReg(AArch64::X18)
1962  .addImm(-8)
1964  }
1965 
1966  return true;
1967 }
1968 
1970  BitVector &SavedRegs,
1971  RegScavenger *RS) const {
1972  // All calls are tail calls in GHC calling conv, and functions have no
1973  // prologue/epilogue.
1975  return;
1976 
1977  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1978  const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
1979  MF.getSubtarget().getRegisterInfo());
1981  unsigned UnspilledCSGPR = AArch64::NoRegister;
1982  unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
1983 
1984  MachineFrameInfo &MFI = MF.getFrameInfo();
1985  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
1986 
1987  unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
1988  ? RegInfo->getBaseRegister()
1989  : (unsigned)AArch64::NoRegister;
1990 
1991  unsigned ExtraCSSpill = 0;
1992  // Figure out which callee-saved registers to save/restore.
1993  for (unsigned i = 0; CSRegs[i]; ++i) {
1994  const unsigned Reg = CSRegs[i];
1995 
1996  // Add the base pointer register to SavedRegs if it is callee-save.
1997  if (Reg == BasePointerReg)
1998  SavedRegs.set(Reg);
1999 
2000  bool RegUsed = SavedRegs.test(Reg);
2001  unsigned PairedReg = CSRegs[i ^ 1];
2002  if (!RegUsed) {
2003  if (AArch64::GPR64RegClass.contains(Reg) &&
2004  !RegInfo->isReservedReg(MF, Reg)) {
2005  UnspilledCSGPR = Reg;
2006  UnspilledCSGPRPaired = PairedReg;
2007  }
2008  continue;
2009  }
2010 
2011  // MachO's compact unwind format relies on all registers being stored in
2012  // pairs.
2013  // FIXME: the usual format is actually better if unwinding isn't needed.
2014  if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
2015  !SavedRegs.test(PairedReg)) {
2016  SavedRegs.set(PairedReg);
2017  if (AArch64::GPR64RegClass.contains(PairedReg) &&
2018  !RegInfo->isReservedReg(MF, PairedReg))
2019  ExtraCSSpill = PairedReg;
2020  }
2021  }
2022 
2023  // Calculates the callee saved stack size.
2024  unsigned CSStackSize = 0;
2026  const MachineRegisterInfo &MRI = MF.getRegInfo();
2027  for (unsigned Reg : SavedRegs.set_bits())
2028  CSStackSize += TRI->getRegSizeInBits(Reg, MRI) / 8;
2029 
2030  // Save number of saved regs, so we can easily update CSStackSize later.
2031  unsigned NumSavedRegs = SavedRegs.count();
2032 
2033  // The frame record needs to be created by saving the appropriate registers
2034  unsigned EstimatedStackSize = MFI.estimateStackSize(MF);
2035  if (hasFP(MF) ||
2036  windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2037  SavedRegs.set(AArch64::FP);
2038  SavedRegs.set(AArch64::LR);
2039  }
2040 
2041  LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
2042  for (unsigned Reg
2043  : SavedRegs.set_bits()) dbgs()
2044  << ' ' << printReg(Reg, RegInfo);
2045  dbgs() << "\n";);
2046 
2047  // If any callee-saved registers are used, the frame cannot be eliminated.
2048  bool CanEliminateFrame = SavedRegs.count() == 0;
2049 
2050  // The CSR spill slots have not been allocated yet, so estimateStackSize
2051  // won't include them.
2052  unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2053  bool BigStack = (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2054  if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2055  AFI->setHasStackFrame(true);
2056 
2057  // Estimate if we might need to scavenge a register at some point in order
2058  // to materialize a stack offset. If so, either spill one additional
2059  // callee-saved register or reserve a special spill slot to facilitate
2060  // register scavenging. If we already spilled an extra callee-saved register
2061  // above to keep the number of spills even, we don't need to do anything else
2062  // here.
2063  if (BigStack) {
2064  if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2065  LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2066  << " to get a scratch register.\n");
2067  SavedRegs.set(UnspilledCSGPR);
2068  // MachO's compact unwind format relies on all registers being stored in
2069  // pairs, so if we need to spill one extra for BigStack, then we need to
2070  // store the pair.
2071  if (produceCompactUnwindFrame(MF))
2072  SavedRegs.set(UnspilledCSGPRPaired);
2073  ExtraCSSpill = UnspilledCSGPRPaired;
2074  }
2075 
2076  // If we didn't find an extra callee-saved register to spill, create
2077  // an emergency spill slot.
2078  if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2079  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2080  const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2081  unsigned Size = TRI->getSpillSize(RC);
2082  unsigned Align = TRI->getSpillAlignment(RC);
2083  int FI = MFI.CreateStackObject(Size, Align, false);
2084  RS->addScavengingFrameIndex(FI);
2085  LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2086  << " as the emergency spill slot.\n");
2087  }
2088  }
2089 
2090  // Adding the size of additional 64bit GPR saves.
2091  CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2092  unsigned AlignedCSStackSize = alignTo(CSStackSize, 16);
2093  LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2094  << EstimatedStackSize + AlignedCSStackSize
2095  << " bytes.\n");
2096 
2097  // Round up to register pair alignment to avoid additional SP adjustment
2098  // instructions.
2099  AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2100  AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2101 }
2102 
2104  const MachineFunction &MF) const {
2105  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2106  return AFI->hasCalleeSaveStackFreeSpace();
2107 }
2108 
2110  MachineFunction &MF, RegScavenger *RS) const {
2111  // If this function isn't doing Win64-style C++ EH, we don't need to do
2112  // anything.
2113  if (!MF.hasEHFunclets())
2114  return;
2115  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2116  MachineFrameInfo &MFI = MF.getFrameInfo();
2117  WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2118 
2119  MachineBasicBlock &MBB = MF.front();
2120  auto MBBI = MBB.begin();
2121  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2122  ++MBBI;
2123 
2124  if (MBBI->isTerminator())
2125  return;
2126 
2127  // Create an UnwindHelp object.
2128  int UnwindHelpFI =
2129  MFI.CreateStackObject(/*size*/8, /*alignment*/16, false);
2130  EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
2131  // We need to store -2 into the UnwindHelp object at the start of the
2132  // function.
2133  DebugLoc DL;
2134  RS->enterBasicBlock(MBB);
2135  unsigned DstReg = RS->scavengeRegister(&AArch64::GPR64RegClass, MBBI, 0);
2136  BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
2137  BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
2138  .addReg(DstReg, getKillRegState(true))
2139  .addFrameIndex(UnwindHelpFI)
2140  .addImm(0);
2141 }
2142 
2143 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP before
2144 /// the update. This is easily retrieved as it is exactly the offset that is set
2145 /// in processFunctionBeforeFrameFinalized.
2147  const MachineFunction &MF, int FI, unsigned &FrameReg,
2148  bool IgnoreSPUpdates) const {
2149  const MachineFrameInfo &MFI = MF.getFrameInfo();
2150  LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
2151  << MFI.getObjectOffset(FI) << "\n");
2152  FrameReg = AArch64::SP;
2153  return MFI.getObjectOffset(FI);
2154 }
2155 
2156 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
2157 /// the parent's frame pointer
2159  const MachineFunction &MF) const {
2160  return 0;
2161 }
2162 
2163 /// Funclets only need to account for space for the callee saved registers,
2164 /// as the locals are accounted for in the parent's stack frame.
2166  const MachineFunction &MF) const {
2167  // This is the size of the pushed CSRs.
2168  unsigned CSSize =
2169  MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
2170  // This is the amount of stack a funclet needs to allocate.
2171  return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
2172  getStackAlignment());
2173 }
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
static cl::opt< bool > ReverseCSRRestoreSeq("reverse-csr-restore-seq", cl::desc("reverse the CSR restore sequence"), cl::init(false), cl::Hidden)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand *> MMOs) const
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:584
BitVector & set()
Definition: BitVector.h:398
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasDebugInfo() const
Returns true if valid debug info is present.
static bool ShouldSignWithAKey(MachineFunction &MF)
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
This class represents lattice values for constants.
Definition: AllocatorList.h:24
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:42
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Definition: StringRef.h:176
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
class llvm::RegisterBankInfo GPR
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool &NeedShadowCallStackProlog)
bool cannotEliminateFrame(const MachineFunction &MF) const
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned getReg() const
getReg - Returns the register number.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset)
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
Definition: MCDwarf.h:488
unsigned Reg
bool test(unsigned Idx) const
Definition: BitVector.h:502
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:321
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:34
F(f)
MachineModuleInfo & getMMI() const
uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:685
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI)
static MCCFIInstruction createDefCfaOffset(MCSymbol *L, int Offset)
.cfi_def_cfa_offset modifies a rule for computing CFA.
Definition: MCDwarf.h:475
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
bool isCallingConvWin64(CallingConv::ID CC) const
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
return AArch64::GPR64RegClass contains(Reg)
static bool isFuncletReturnInstr(const MachineInstr &MI)
static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
unsigned getFrameRegister(const MachineFunction &MF) const override
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
unsigned getSpillAlignment(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class...
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:690
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
const HexagonInstrInfo * TII
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
Printable printReg(unsigned Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required, we reserve argument space for call sites in the function immediately on entry to the current function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
This file contains the simple types necessary to represent the attributes associated with functions a...
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool InProlog=true)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:409
int getDwarfRegNum(unsigned RegNum, bool isEH) const
Map a target register to an equivalent dwarf register number.
LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
int getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, unsigned &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent&#39;s frame pointer...
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:267
Key
PAL metadata keys.
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
virtual void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, bool KillSrc) const
Emit instructions to copy a pair of physical registers.
const AArch64RegisterInfo * getRegisterInfo() const override
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
const MCContext & getContext() const
iterator getLastNonDebugInstr()
Returns an iterator to the last non-debug instruction in the basic block, or end().
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:224
bool hasPersonalityFn() const
Check whether this function has a personality function.
Definition: Function.h:702
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
virtual const TargetInstrInfo * getInstrInfo() const
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
bool isXRegisterReserved(size_t i) const
Analysis containing CSE Info
Definition: CSEInfo.cpp:21
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
unsigned getKillRegState(bool B)
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:118
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
TargetInstrInfo - Interface to description of machine instruction set.
unsigned getDefRegState(bool B)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
const AArch64TargetLowering * getTargetLowering() const override
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1166
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:423
static MCCFIInstruction createNegateRAState(MCSymbol *L)
.cfi_negate_ra_state AArch64 negate RA state.
Definition: MCDwarf.h:514
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
This file declares the machine register scavenger class.
MCSymbol * createTempSymbol(bool CanBeUnnamed=true)
Create and return a new assembler temporary symbol with a unique but unspecified name.
Definition: MCContext.cpp:217
unsigned const MachineRegisterInfo * MRI
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:120
static MCCFIInstruction createDefCfa(MCSymbol *L, unsigned Register, int Offset)
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it...
Definition: MCDwarf.h:461
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static bool ShouldSignReturnAddress(MachineFunction &MF)
bool hasEHFunclets() const
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size FIXME: The defaults need to be removed once all of the backends/clients are updat...
Definition: DataLayout.cpp:629
int resolveFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg, bool PreferFP=false) const
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any DBG_VALUE and DBG_LABEL instructions...
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
unsigned getMaxAlignment() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
void setImm(int64_t immVal)
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, unsigned LocalStackSize, bool NeedsWinCFI)
bool hasLocalEscape() const
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const MachineBasicBlock & front() const
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
virtual bool hasReservedCallFrame(const MachineFunction &MF) const
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required, we reserve argument space for call sites in the function immediately on entry to the current function.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const
The memory access writes data.
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:497
size_type count() const
count - Returns the number of bits which are set.
Definition: BitVector.h:173
static void InsertReturnAddressAuth(MachineFunction &MF, MachineBasicBlock &MBB)
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:213
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
unsigned estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
Information about stack frame layout on the target.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling...
int64_t getImm() const
const Function & getFunction() const
Return the LLVM function that this machine code represents.
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
Definition: Function.h:573
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:941
bool isPhysRegUsed(unsigned PhysReg) const
Return true if the specified register is modified or read in this function.
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
CodeModel::Model getCodeModel() const
Returns the code model.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
static bool needsWinCFI(const MachineFunction &MF)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:169
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static bool windowsRequiresStackProbe(MachineFunction &MF, unsigned StackSizeInBytes)
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register...
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:686
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:254
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
TargetSubtargetInfo - Generic base class for all target subtargets.
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
bool isReservedReg(const MachineFunction &MF, unsigned Reg) const
bool isLiveIn(unsigned Reg) const
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
static void adaptForLdStOpt(MachineBasicBlock &MBB, MachineBasicBlock::iterator FirstSPPopI, MachineBasicBlock::iterator LastPopI)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
void enterBasicBlock(MachineBasicBlock &MBB)
Start tracking liveness from the begin of basic block MBB.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:49
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:195
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
TargetOptions Options
Definition: TargetMachine.h:97
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned char TargetFlags=0) const
#define I(x, y, z)
Definition: MD5.cpp:58
virtual const TargetFrameLowering * getFrameLowering() const
unsigned scavengeRegister(const TargetRegisterClass *RC, MachineBasicBlock::iterator I, int SPAdj)
Make a register of the specific register class available and do the appropriate bookkeeping.
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
uint32_t Size
Definition: Profile.cpp:47
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
static bool produceCompactUnwindFrame(MachineFunction &MF)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
bool hasBasePointer(const MachineFunction &MF) const
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:130
bool needsStackRealignment(const MachineFunction &MF) const
True if storage within the function requires the stack pointer to be aligned more than the normal cal...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1299
const AArch64InstrInfo * getInstrInfo() const override
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:251
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
Definition: LivePhysRegs.h:80
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:331
bool isMaxCallFrameSizeComputed() const
IRTranslator LLVM IR MI
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
unsigned getRegSizeInBits(const TargetRegisterClass &RC) const
Return the size in bits of a register from class RC.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals)
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition: MCDwarf.h:549
#define LLVM_DEBUG(X)
Definition: Debug.h:123
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:414
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects...
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
void setCalleeSavedStackSize(unsigned Size)
This class contains meta information specific to a module.
bool hasCalls() const
Return true if the current function has any function calls.