LCOV - code coverage report
Current view: top level - lib/Target/AArch64 - AArch64FrameLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 439 496 88.5 %
Date: 2018-10-20 13:21:21 Functions: 23 26 88.5 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the AArch64 implementation of TargetFrameLowering class.
      11             : //
      12             : // On AArch64, stack frames are structured as follows:
      13             : //
      14             : // The stack grows downward.
      15             : //
      16             : // All of the individual frame areas on the frame below are optional, i.e. it's
      17             : // possible to create a function so that the particular area isn't present
      18             : // in the frame.
      19             : //
      20             : // At function entry, the "frame" looks as follows:
      21             : //
      22             : // |                                   | Higher address
      23             : // |-----------------------------------|
      24             : // |                                   |
      25             : // | arguments passed on the stack     |
      26             : // |                                   |
      27             : // |-----------------------------------| <- sp
      28             : // |                                   | Lower address
      29             : //
      30             : //
      31             : // After the prologue has run, the frame has the following general structure.
      32             : // Note that this doesn't depict the case where a red-zone is used. Also,
      33             : // technically the last frame area (VLAs) doesn't get created until in the
      34             : // main function body, after the prologue is run. However, it's depicted here
      35             : // for completeness.
      36             : //
      37             : // |                                   | Higher address
      38             : // |-----------------------------------|
      39             : // |                                   |
      40             : // | arguments passed on the stack     |
      41             : // |                                   |
      42             : // |-----------------------------------|
      43             : // |                                   |
      44             : // | (Win64 only) varargs from reg     |
      45             : // |                                   |
      46             : // |-----------------------------------|
      47             : // |                                   |
      48             : // | prev_fp, prev_lr                  |
      49             : // | (a.k.a. "frame record")           |
      50             : // |-----------------------------------| <- fp(=x29)
      51             : // |                                   |
      52             : // | other callee-saved registers      |
      53             : // |                                   |
      54             : // |-----------------------------------|
      55             : // |.empty.space.to.make.part.below....|
      56             : // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
      57             : // |.the.standard.16-byte.alignment....|  compile time; if present)
      58             : // |-----------------------------------|
      59             : // |                                   |
      60             : // | local variables of fixed size     |
      61             : // | including spill slots             |
      62             : // |-----------------------------------| <- bp(not defined by ABI,
      63             : // |.variable-sized.local.variables....|       LLVM chooses X19)
      64             : // |.(VLAs)............................| (size of this area is unknown at
      65             : // |...................................|  compile time)
      66             : // |-----------------------------------| <- sp
      67             : // |                                   | Lower address
      68             : //
      69             : //
      70             : // To access the data in a frame, at-compile time, a constant offset must be
      71             : // computable from one of the pointers (fp, bp, sp) to access it. The size
      72             : // of the areas with a dotted background cannot be computed at compile-time
      73             : // if they are present, making it required to have all three of fp, bp and
      74             : // sp to be set up to be able to access all contents in the frame areas,
      75             : // assuming all of the frame areas are non-empty.
      76             : //
      77             : // For most functions, some of the frame areas are empty. For those functions,
      78             : // it may not be necessary to set up fp or bp:
      79             : // * A base pointer is definitely needed when there are both VLAs and local
      80             : //   variables with more-than-default alignment requirements.
      81             : // * A frame pointer is definitely needed when there are local variables with
      82             : //   more-than-default alignment requirements.
      83             : //
      84             : // In some cases when a base pointer is not strictly needed, it is generated
      85             : // anyway when offsets from the frame pointer to access local variables become
      86             : // so large that the offset can't be encoded in the immediate fields of loads
      87             : // or stores.
      88             : //
      89             : // FIXME: also explain the redzone concept.
      90             : // FIXME: also explain the concept of reserved call frames.
      91             : //
      92             : //===----------------------------------------------------------------------===//
      93             : 
      94             : #include "AArch64FrameLowering.h"
      95             : #include "AArch64InstrInfo.h"
      96             : #include "AArch64MachineFunctionInfo.h"
      97             : #include "AArch64RegisterInfo.h"
      98             : #include "AArch64Subtarget.h"
      99             : #include "AArch64TargetMachine.h"
     100             : #include "MCTargetDesc/AArch64AddressingModes.h"
     101             : #include "llvm/ADT/ScopeExit.h"
     102             : #include "llvm/ADT/SmallVector.h"
     103             : #include "llvm/ADT/Statistic.h"
     104             : #include "llvm/CodeGen/LivePhysRegs.h"
     105             : #include "llvm/CodeGen/MachineBasicBlock.h"
     106             : #include "llvm/CodeGen/MachineFrameInfo.h"
     107             : #include "llvm/CodeGen/MachineFunction.h"
     108             : #include "llvm/CodeGen/MachineInstr.h"
     109             : #include "llvm/CodeGen/MachineInstrBuilder.h"
     110             : #include "llvm/CodeGen/MachineMemOperand.h"
     111             : #include "llvm/CodeGen/MachineModuleInfo.h"
     112             : #include "llvm/CodeGen/MachineOperand.h"
     113             : #include "llvm/CodeGen/MachineRegisterInfo.h"
     114             : #include "llvm/CodeGen/RegisterScavenging.h"
     115             : #include "llvm/CodeGen/TargetInstrInfo.h"
     116             : #include "llvm/CodeGen/TargetRegisterInfo.h"
     117             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
     118             : #include "llvm/IR/Attributes.h"
     119             : #include "llvm/IR/CallingConv.h"
     120             : #include "llvm/IR/DataLayout.h"
     121             : #include "llvm/IR/DebugLoc.h"
     122             : #include "llvm/IR/Function.h"
     123             : #include "llvm/MC/MCDwarf.h"
     124             : #include "llvm/Support/CommandLine.h"
     125             : #include "llvm/Support/Debug.h"
     126             : #include "llvm/Support/ErrorHandling.h"
     127             : #include "llvm/Support/MathExtras.h"
     128             : #include "llvm/Support/raw_ostream.h"
     129             : #include "llvm/Target/TargetMachine.h"
     130             : #include "llvm/Target/TargetOptions.h"
     131             : #include <cassert>
     132             : #include <cstdint>
     133             : #include <iterator>
     134             : #include <vector>
     135             : 
     136             : using namespace llvm;
     137             : 
     138             : #define DEBUG_TYPE "frame-info"
     139             : 
     140             : static cl::opt<bool> EnableRedZone("aarch64-redzone",
     141             :                                    cl::desc("enable use of redzone on AArch64"),
     142             :                                    cl::init(false), cl::Hidden);
     143             : 
     144             : static cl::opt<bool>
     145             :     ReverseCSRRestoreSeq("reverse-csr-restore-seq",
     146             :                          cl::desc("reverse the CSR restore sequence"),
     147             :                          cl::init(false), cl::Hidden);
     148             : 
     149             : STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
     150             : 
     151             : /// This is the biggest offset to the stack pointer we can encode in aarch64
     152             : /// instructions (without using a separate calculation and a temp register).
     153             : /// Note that the exception here are vector stores/loads which cannot encode any
     154             : /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
     155             : static const unsigned DefaultSafeSPDisplacement = 255;
     156             : 
     157             : /// Look at each instruction that references stack frames and return the stack
     158             : /// size limit beyond which some of these instructions will require a scratch
     159             : /// register during their expansion later.
     160       15047 : static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
     161             :   // FIXME: For now, just conservatively guestimate based on unscaled indexing
     162             :   // range. We'll end up allocating an unnecessary spill slot a lot, but
     163             :   // realistically that's not a big deal at this stage of the game.
     164       33032 :   for (MachineBasicBlock &MBB : MF) {
     165       95161 :     for (MachineInstr &MI : MBB) {
     166       77077 :       if (MI.isDebugInstr() || MI.isPseudo() ||
     167       69702 :           MI.getOpcode() == AArch64::ADDXri ||
     168             :           MI.getOpcode() == AArch64::ADDSXri)
     169        8301 :         continue;
     170             : 
     171      256971 :       for (const MachineOperand &MO : MI.operands()) {
     172      188104 :         if (!MO.isFI())
     173      184407 :           continue;
     174             : 
     175        3697 :         int Offset = 0;
     176        3697 :         if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
     177             :             AArch64FrameOffsetCannotUpdate)
     178           8 :           return 0;
     179             :       }
     180             :     }
     181             :   }
     182             :   return DefaultSafeSPDisplacement;
     183             : }
     184             : 
     185       19214 : bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
     186       19214 :   if (!EnableRedZone)
     187             :     return false;
     188             :   // Don't use the red zone if the function explicitly asks us not to.
     189             :   // This is typically used for kernel code.
     190          94 :   if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
     191             :     return false;
     192             : 
     193          94 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     194             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     195          94 :   unsigned NumBytes = AFI->getLocalStackSize();
     196             : 
     197          94 :   return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
     198             : }
     199             : 
     200             : /// hasFP - Return true if the specified function should have a dedicated frame
     201             : /// pointer register.
     202      294057 : bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
     203      294057 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     204      294057 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     205             :   // Retain behavior of always omitting the FP for leaf functions when possible.
     206      294057 :   if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
     207             :     return true;
     208      288772 :   if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
     209      865908 :       MFI.hasStackMap() || MFI.hasPatchPoint() ||
     210      287253 :       RegInfo->needsStackRealignment(MF))
     211        3199 :     return true;
     212             :   // With large callframes around we may need to use FP to access the scavenging
     213             :   // emergency spillslot.
     214             :   //
     215             :   // Unfortunately some calls to hasFP() like machine verifier ->
     216             :   // getReservedReg() -> hasFP in the middle of global isel are too early
     217             :   // to know the max call frame size. Hopefully conservatively returning "true"
     218             :   // in those cases is fine.
     219             :   // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
     220      286800 :   if (!MFI.isMaxCallFrameSizeComputed() ||
     221             :       MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
     222       18565 :     return true;
     223             : 
     224             :   return false;
     225             : }
     226             : 
     227             : /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
     228             : /// not required, we reserve argument space for call sites in the function
     229             : /// immediately on entry to the current function.  This eliminates the need for
     230             : /// add/sub sp brackets around call sites.  Returns true if the call frame is
     231             : /// included as part of the stack frame.
     232             : bool
     233       10297 : AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
     234       10297 :   return !MF.getFrameInfo().hasVarSizedObjects();
     235             : }
     236             : 
     237        3906 : MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
     238             :     MachineFunction &MF, MachineBasicBlock &MBB,
     239             :     MachineBasicBlock::iterator I) const {
     240             :   const AArch64InstrInfo *TII =
     241        3906 :       static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
     242             :   DebugLoc DL = I->getDebugLoc();
     243        3906 :   unsigned Opc = I->getOpcode();
     244        3906 :   bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
     245        3906 :   uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
     246             : 
     247        3906 :   const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
     248        3906 :   if (!TFI->hasReservedCallFrame(MF)) {
     249         136 :     unsigned Align = getStackAlignment();
     250             : 
     251         136 :     int64_t Amount = I->getOperand(0).getImm();
     252         136 :     Amount = alignTo(Amount, Align);
     253         136 :     if (!IsDestroy)
     254          68 :       Amount = -Amount;
     255             : 
     256             :     // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
     257             :     // doesn't have to pop anything), then the first operand will be zero too so
     258             :     // this adjustment is a no-op.
     259         136 :     if (CalleePopAmount == 0) {
     260             :       // FIXME: in-function stack adjustment for calls is limited to 24-bits
     261             :       // because there's no guaranteed temporary register available.
     262             :       //
     263             :       // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
     264             :       // 1) For offset <= 12-bit, we use LSL #0
     265             :       // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
     266             :       // LSL #0, and the other uses LSL #12.
     267             :       //
     268             :       // Most call frames will be allocated at the start of a function so
     269             :       // this is OK, but it is a limitation that needs dealing with.
     270             :       assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
     271         135 :       emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
     272             :     }
     273        3770 :   } else if (CalleePopAmount != 0) {
     274             :     // If the calling convention demands that the callee pops arguments from the
     275             :     // stack, we want to add it back if we have a reserved call frame.
     276             :     assert(CalleePopAmount < 0xffffff && "call frame too large");
     277          12 :     emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
     278             :                     TII);
     279             :   }
     280        3906 :   return MBB.erase(I);
     281             : }
     282             : 
     283       29816 : static bool ShouldSignReturnAddress(MachineFunction &MF) {
     284             :   // The function should be signed in the following situations:
     285             :   // - sign-return-address=all
     286             :   // - sign-return-address=non-leaf and the functions spills the LR
     287             : 
     288       29816 :   const Function &F = MF.getFunction();
     289       29816 :   if (!F.hasFnAttribute("sign-return-address"))
     290             :     return false;
     291             : 
     292          16 :   StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
     293             :   if (Scope.equals("none"))
     294             :     return false;
     295             : 
     296             :   if (Scope.equals("all"))
     297             :     return true;
     298             : 
     299             :   assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
     300             : 
     301           6 :   for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
     302           4 :     if (Info.getReg() == AArch64::LR)
     303             :       return true;
     304             : 
     305             :   return false;
     306             : }
     307             : 
     308        1054 : void AArch64FrameLowering::emitCalleeSavedFrameMoves(
     309             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
     310        1054 :   MachineFunction &MF = *MBB.getParent();
     311        1054 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     312        1054 :   const TargetSubtargetInfo &STI = MF.getSubtarget();
     313        1054 :   const MCRegisterInfo *MRI = STI.getRegisterInfo();
     314        1054 :   const TargetInstrInfo *TII = STI.getInstrInfo();
     315             :   DebugLoc DL = MBB.findDebugLoc(MBBI);
     316             : 
     317             :   // Add callee saved registers to move list.
     318             :   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
     319        1054 :   if (CSI.empty())
     320             :     return;
     321             : 
     322        3854 :   for (const auto &Info : CSI) {
     323        2800 :     unsigned Reg = Info.getReg();
     324             :     int64_t Offset =
     325        2800 :         MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
     326        2800 :     unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
     327             :     unsigned CFIIndex = MF.addFrameInst(
     328        2800 :         MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     329        5600 :     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     330             :         .addCFIIndex(CFIIndex)
     331             :         .setMIFlags(MachineInstr::FrameSetup);
     332             :   }
     333             : }
     334             : 
     335             : // Find a scratch register that we can use at the start of the prologue to
     336             : // re-align the stack pointer.  We avoid using callee-save registers since they
     337             : // may appear to be free when this is called from canUseAsPrologue (during
     338             : // shrink wrapping), but then no longer be free when this is called from
     339             : // emitPrologue.
     340             : //
     341             : // FIXME: This is a bit conservative, since in the above case we could use one
     342             : // of the callee-save registers as a scratch temp to re-align the stack pointer,
     343             : // but we would then have to make sure that we were in fact saving at least one
     344             : // callee-save register in the prologue, which is additional complexity that
     345             : // doesn't seem worth the benefit.
     346          27 : static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
     347          27 :   MachineFunction *MF = MBB->getParent();
     348             : 
     349             :   // If MBB is an entry block, use X9 as the scratch register
     350          27 :   if (&MF->front() == MBB)
     351             :     return AArch64::X9;
     352             : 
     353           9 :   const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
     354             :   const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
     355           9 :   LivePhysRegs LiveRegs(TRI);
     356           9 :   LiveRegs.addLiveIns(*MBB);
     357             : 
     358             :   // Mark callee saved registers as used so we will not choose them.
     359           9 :   const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
     360         189 :   for (unsigned i = 0; CSRegs[i]; ++i)
     361         180 :     LiveRegs.addReg(CSRegs[i]);
     362             : 
     363             :   // Prefer X9 since it was historically used for the prologue scratch reg.
     364           9 :   const MachineRegisterInfo &MRI = MF->getRegInfo();
     365           9 :   if (LiveRegs.available(MRI, AArch64::X9))
     366             :     return AArch64::X9;
     367             : 
     368          37 :   for (unsigned Reg : AArch64::GPR64RegClass) {
     369          36 :     if (LiveRegs.available(MRI, Reg))
     370           2 :       return Reg;
     371             :   }
     372             :   return AArch64::NoRegister;
     373             : }
     374             : 
     375          78 : bool AArch64FrameLowering::canUseAsPrologue(
     376             :     const MachineBasicBlock &MBB) const {
     377          78 :   const MachineFunction *MF = MBB.getParent();
     378             :   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
     379          78 :   const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
     380             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     381             : 
     382             :   // Don't need a scratch register if we're not going to re-align the stack.
     383          78 :   if (!RegInfo->needsStackRealignment(*MF))
     384             :     return true;
     385             :   // Otherwise, we can use any block as long as it has a scratch register
     386             :   // available.
     387           7 :   return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
     388             : }
     389             : 
     390       30858 : static bool windowsRequiresStackProbe(MachineFunction &MF,
     391             :                                       unsigned StackSizeInBytes) {
     392       30858 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     393       30858 :   if (!Subtarget.isTargetWindows())
     394             :     return false;
     395         142 :   const Function &F = MF.getFunction();
     396             :   // TODO: When implementing stack protectors, take that into account
     397             :   // for the probe threshold.
     398         142 :   unsigned StackProbeSize = 4096;
     399         142 :   if (F.hasFnAttribute("stack-probe-size"))
     400           0 :     F.getFnAttribute("stack-probe-size")
     401           0 :         .getValueAsString()
     402           0 :         .getAsInteger(0, StackProbeSize);
     403         142 :   return (StackSizeInBytes >= StackProbeSize) &&
     404           6 :          !F.hasFnAttribute("no-stack-arg-probe");
     405             : }
     406             : 
     407       16387 : bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
     408             :     MachineFunction &MF, unsigned StackBumpBytes) const {
     409       16387 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     410       16387 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     411       16387 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     412             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     413             : 
     414       16387 :   if (AFI->getLocalStackSize() == 0)
     415             :     return false;
     416             : 
     417             :   // 512 is the maximum immediate for stp/ldp that will be used for
     418             :   // callee-save save/restores
     419        1373 :   if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
     420          50 :     return false;
     421             : 
     422        1323 :   if (MFI.hasVarSizedObjects())
     423             :     return false;
     424             : 
     425        1289 :   if (RegInfo->needsStackRealignment(MF))
     426             :     return false;
     427             : 
     428             :   // This isn't strictly necessary, but it simplifies things a bit since the
     429             :   // current RedZone handling code assumes the SP is adjusted by the
     430             :   // callee-save save/restore code.
     431        1265 :   if (canUseRedZone(MF))
     432           5 :     return false;
     433             : 
     434             :   return true;
     435             : }
     436             : 
     437             : // Convert callee-save register save/restore instruction to do stack pointer
     438             : // decrement/increment to allocate/deallocate the callee-save stack area by
     439             : // converting store/load to use pre/post increment version.
     440        1698 : static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     441             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     442             :     const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc) {
     443             :   // Ignore instructions that do not operate on SP, i.e. shadow call stack
     444             :   // instructions.
     445        3400 :   while (MBBI->getOpcode() == AArch64::STRXpost ||
     446             :          MBBI->getOpcode() == AArch64::LDRXpre) {
     447             :     assert(MBBI->getOperand(0).getReg() != AArch64::SP);
     448             :     ++MBBI;
     449             :   }
     450             : 
     451             :   unsigned NewOpc;
     452             :   int Scale = 1;
     453        1698 :   switch (MBBI->getOpcode()) {
     454           0 :   default:
     455           0 :     llvm_unreachable("Unexpected callee-save save/restore opcode!");
     456             :   case AArch64::STPXi:
     457             :     NewOpc = AArch64::STPXpre;
     458             :     Scale = 8;
     459             :     break;
     460          39 :   case AArch64::STPDi:
     461             :     NewOpc = AArch64::STPDpre;
     462             :     Scale = 8;
     463          39 :     break;
     464           4 :   case AArch64::STPQi:
     465             :     NewOpc = AArch64::STPQpre;
     466             :     Scale = 16;
     467           4 :     break;
     468         341 :   case AArch64::STRXui:
     469             :     NewOpc = AArch64::STRXpre;
     470         341 :     break;
     471          43 :   case AArch64::STRDui:
     472             :     NewOpc = AArch64::STRDpre;
     473          43 :     break;
     474           0 :   case AArch64::STRQui:
     475             :     NewOpc = AArch64::STRQpre;
     476           0 :     break;
     477         414 :   case AArch64::LDPXi:
     478             :     NewOpc = AArch64::LDPXpost;
     479             :     Scale = 8;
     480         414 :     break;
     481          39 :   case AArch64::LDPDi:
     482             :     NewOpc = AArch64::LDPDpost;
     483             :     Scale = 8;
     484          39 :     break;
     485           0 :   case AArch64::LDPQi:
     486             :     NewOpc = AArch64::LDPQpost;
     487             :     Scale = 16;
     488           0 :     break;
     489         351 :   case AArch64::LDRXui:
     490             :     NewOpc = AArch64::LDRXpost;
     491         351 :     break;
     492          43 :   case AArch64::LDRDui:
     493             :     NewOpc = AArch64::LDRDpost;
     494          43 :     break;
     495           0 :   case AArch64::LDRQui:
     496             :     NewOpc = AArch64::LDRQpost;
     497           0 :     break;
     498             :   }
     499             : 
     500        3396 :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
     501        1698 :   MIB.addReg(AArch64::SP, RegState::Define);
     502             : 
     503             :   // Copy all operands other than the immediate offset.
     504             :   unsigned OpndIdx = 0;
     505        6014 :   for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
     506             :        ++OpndIdx)
     507        4316 :     MIB.add(MBBI->getOperand(OpndIdx));
     508             : 
     509             :   assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
     510             :          "Unexpected immediate offset in first/last callee-save save/restore "
     511             :          "instruction!");
     512             :   assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
     513             :          "Unexpected base register in callee-save save/restore instruction!");
     514             :   assert(CSStackSizeInc % Scale == 0);
     515        1698 :   MIB.addImm(CSStackSizeInc / Scale);
     516             : 
     517        1698 :   MIB.setMIFlags(MBBI->getFlags());
     518        1698 :   MIB.setMemRefs(MBBI->memoperands());
     519             : 
     520        1698 :   return std::prev(MBB.erase(MBBI));
     521             : }
     522             : 
     523             : // Fixup callee-save register save/restore instructions to take into account
     524             : // combined SP bump by adding the local stack size to the stack offsets.
     525        2036 : static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
     526             :                                               unsigned LocalStackSize) {
     527        2036 :   unsigned Opc = MI.getOpcode();
     528             : 
     529             :   // Ignore instructions that do not operate on SP, i.e. shadow call stack
     530             :   // instructions.
     531        2036 :   if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre) {
     532             :     assert(MI.getOperand(0).getReg() != AArch64::SP);
     533             :     return;
     534             :   }
     535             : 
     536             :   unsigned Scale;
     537        2036 :   switch (Opc) {
     538             :   case AArch64::STPXi:
     539             :   case AArch64::STRXui:
     540             :   case AArch64::STPDi:
     541             :   case AArch64::STRDui:
     542             :   case AArch64::LDPXi:
     543             :   case AArch64::LDRXui:
     544             :   case AArch64::LDPDi:
     545             :   case AArch64::LDRDui:
     546             :     Scale = 8;
     547             :     break;
     548           8 :   case AArch64::STPQi:
     549             :   case AArch64::STRQui:
     550             :   case AArch64::LDPQi:
     551             :   case AArch64::LDRQui:
     552             :     Scale = 16;
     553           8 :     break;
     554           0 :   default:
     555           0 :     llvm_unreachable("Unexpected callee-save save/restore opcode!");
     556             :   }
     557             : 
     558        2036 :   unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
     559             :   assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
     560             :          "Unexpected base register in callee-save save/restore instruction!");
     561             :   // Last operand is immediate offset that needs fixing.
     562        2036 :   MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
     563             :   // All generated opcodes have scaled offsets.
     564             :   assert(LocalStackSize % Scale == 0);
     565        2036 :   OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
     566             : }
     567             : 
     568             : static void adaptForLdStOpt(MachineBasicBlock &MBB,
     569             :                             MachineBasicBlock::iterator FirstSPPopI,
     570             :                             MachineBasicBlock::iterator LastPopI) {
     571             :   // Sometimes (when we restore in the same order as we save), we can end up
     572             :   // with code like this:
     573             :   //
     574             :   // ldp      x26, x25, [sp]
     575             :   // ldp      x24, x23, [sp, #16]
     576             :   // ldp      x22, x21, [sp, #32]
     577             :   // ldp      x20, x19, [sp, #48]
     578             :   // add      sp, sp, #64
     579             :   //
     580             :   // In this case, it is always better to put the first ldp at the end, so
     581             :   // that the load-store optimizer can run and merge the ldp and the add into
     582             :   // a post-index ldp.
     583             :   // If we managed to grab the first pop instruction, move it to the end.
     584       14036 :   if (ReverseCSRRestoreSeq)
     585           4 :     MBB.splice(FirstSPPopI, &MBB, LastPopI);
     586             :   // We should end up with something like this now:
     587             :   //
     588             :   // ldp      x24, x23, [sp, #16]
     589             :   // ldp      x22, x21, [sp, #32]
     590             :   // ldp      x20, x19, [sp, #48]
     591             :   // ldp      x26, x25, [sp]
     592             :   // add      sp, sp, #64
     593             :   //
     594             :   // and the load-store optimizer can merge the last two instructions into:
     595             :   //
     596             :   // ldp      x26, x25, [sp], #64
     597             :   //
     598             : }
     599             : 
     600       14794 : void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
     601             :                                         MachineBasicBlock &MBB) const {
     602             :   MachineBasicBlock::iterator MBBI = MBB.begin();
     603       14794 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     604       14794 :   const Function &F = MF.getFunction();
     605       14794 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     606             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     607             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     608       14794 :   MachineModuleInfo &MMI = MF.getMMI();
     609       14794 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     610       14794 :   bool needsFrameMoves = MMI.hasDebugInfo() || F.needsUnwindTableEntry();
     611       14794 :   bool HasFP = hasFP(MF);
     612             : 
     613             :   // At this point, we're going to decide whether or not the function uses a
     614             :   // redzone. In most cases, the function doesn't have a redzone so let's
     615             :   // assume that's false and set it to true in the case that there's a redzone.
     616             :   AFI->setHasRedZone(false);
     617             : 
     618             :   // Debug location must be unknown since the first debug location is used
     619             :   // to determine the end of the prologue.
     620       14794 :   DebugLoc DL;
     621             : 
     622       14794 :   if (ShouldSignReturnAddress(MF)) {
     623          12 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
     624             :         .setMIFlag(MachineInstr::FrameSetup);
     625             :   }
     626             : 
     627             :   // All calls are tail calls in GHC calling conv, and functions have no
     628             :   // prologue/epilogue.
     629       29588 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     630             :     return;
     631             : 
     632       14788 :   int NumBytes = (int)MFI.getStackSize();
     633       14788 :   if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
     634             :     assert(!HasFP && "unexpected function without stack frame but with FP");
     635             : 
     636             :     // All of the stack allocation is for locals.
     637             :     AFI->setLocalStackSize(NumBytes);
     638             : 
     639       13423 :     if (!NumBytes)
     640             :       return;
     641             :     // REDZONE: If the stack size is less than 128 bytes, we don't need
     642             :     // to actually allocate.
     643         225 :     if (canUseRedZone(MF)) {
     644             :       AFI->setHasRedZone(true);
     645             :       ++NumRedZoneFunctions;
     646             :     } else {
     647         222 :       emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
     648             :                       MachineInstr::FrameSetup);
     649             : 
     650             :       // Label used to tie together the PROLOG_LABEL and the MachineMoves.
     651         222 :       MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
     652             :       // Encode the stack size of the leaf function.
     653             :       unsigned CFIIndex = MF.addFrameInst(
     654         222 :           MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
     655         444 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     656             :           .addCFIIndex(CFIIndex)
     657             :           .setMIFlags(MachineInstr::FrameSetup);
     658             :     }
     659         225 :     return;
     660             :   }
     661             : 
     662             :   bool IsWin64 =
     663        1365 :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
     664        1299 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
     665             : 
     666        1365 :   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
     667             :   // All of the remaining stack allocations are for locals.
     668        1365 :   AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
     669             : 
     670        1365 :   bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
     671        1365 :   if (CombineSPBump) {
     672         514 :     emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
     673             :                     MachineInstr::FrameSetup);
     674             :     NumBytes = 0;
     675         851 :   } else if (PrologueSaveSize != 0) {
     676             :     MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(MBB, MBBI, DL, TII,
     677         851 :                                                      -PrologueSaveSize);
     678         851 :     NumBytes -= PrologueSaveSize;
     679             :   }
     680             :   assert(NumBytes >= 0 && "Negative stack allocation size!?");
     681             : 
     682             :   // Move past the saves of the callee-saved registers, fixing up the offsets
     683             :   // and pre-inc if we decided to combine the callee-save and local stack
     684             :   // pointer bump above.
     685             :   MachineBasicBlock::iterator End = MBB.end();
     686        3771 :   while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
     687        2406 :     if (CombineSPBump)
     688        1020 :       fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize());
     689             :     ++MBBI;
     690             :   }
     691        1365 :   if (HasFP) {
     692             :     // Only set up FP if we actually need to. Frame pointer is fp =
     693             :     // sp - fixedobject - 16.
     694         293 :     int FPOffset = AFI->getCalleeSavedStackSize() - 16;
     695         293 :     if (CombineSPBump)
     696         104 :       FPOffset += AFI->getLocalStackSize();
     697             : 
     698             :     // Issue    sub fp, sp, FPOffset or
     699             :     //          mov fp,sp          when FPOffset is zero.
     700             :     // Note: All stores of callee-saved registers are marked as "FrameSetup".
     701             :     // This code marks the instruction(s) that set the FP also.
     702         293 :     emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
     703             :                     MachineInstr::FrameSetup);
     704             :   }
     705             : 
     706        1365 :   if (windowsRequiresStackProbe(MF, NumBytes)) {
     707           2 :     uint32_t NumWords = NumBytes >> 4;
     708             : 
     709           4 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
     710           2 :         .addImm(NumWords)
     711             :         .setMIFlags(MachineInstr::FrameSetup);
     712             : 
     713           2 :     switch (MF.getTarget().getCodeModel()) {
     714           1 :     case CodeModel::Tiny:
     715             :     case CodeModel::Small:
     716             :     case CodeModel::Medium:
     717             :     case CodeModel::Kernel:
     718           2 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
     719             :           .addExternalSymbol("__chkstk")
     720           1 :           .addReg(AArch64::X15, RegState::Implicit)
     721             :           .setMIFlags(MachineInstr::FrameSetup);
     722           1 :       break;
     723           1 :     case CodeModel::Large:
     724           2 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
     725           1 :           .addReg(AArch64::X16, RegState::Define)
     726             :           .addExternalSymbol("__chkstk")
     727             :           .addExternalSymbol("__chkstk")
     728             :           .setMIFlags(MachineInstr::FrameSetup);
     729             : 
     730           2 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
     731           1 :           .addReg(AArch64::X16, RegState::Kill)
     732           1 :           .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
     733             :           .setMIFlags(MachineInstr::FrameSetup);
     734           1 :       break;
     735             :     }
     736             : 
     737           6 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
     738           2 :         .addReg(AArch64::SP, RegState::Kill)
     739           2 :         .addReg(AArch64::X15, RegState::Kill)
     740             :         .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
     741             :         .setMIFlags(MachineInstr::FrameSetup);
     742             :     NumBytes = 0;
     743             :   }
     744             : 
     745             :   // Allocate space for the rest of the frame.
     746        1363 :   if (NumBytes) {
     747          54 :     const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
     748             :     unsigned scratchSPReg = AArch64::SP;
     749             : 
     750          54 :     if (NeedsRealignment) {
     751          20 :       scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
     752             :       assert(scratchSPReg != AArch64::NoRegister);
     753             :     }
     754             : 
     755             :     // If we're a leaf function, try using the red zone.
     756          54 :     if (!canUseRedZone(MF))
     757             :       // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
     758             :       // the correct value here, as NumBytes also includes padding bytes,
     759             :       // which shouldn't be counted here.
     760          53 :       emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
     761             :                       MachineInstr::FrameSetup);
     762             : 
     763          54 :     if (NeedsRealignment) {
     764          20 :       const unsigned Alignment = MFI.getMaxAlignment();
     765          20 :       const unsigned NrBitsToZero = countTrailingZeros(Alignment);
     766             :       assert(NrBitsToZero > 1);
     767             :       assert(scratchSPReg != AArch64::SP);
     768             : 
     769             :       // SUB X9, SP, NumBytes
     770             :       //   -- X9 is temporary register, so shouldn't contain any live data here,
     771             :       //   -- free to use. This is already produced by emitFrameOffset above.
     772             :       // AND SP, X9, 0b11111...0000
     773             :       // The logical immediates have a non-trivial encoding. The following
     774             :       // formula computes the encoded immediate with all ones but
     775             :       // NrBitsToZero zero bits as least significant bits.
     776          20 :       uint32_t andMaskEncoded = (1 << 12)                         // = N
     777          20 :                                 | ((64 - NrBitsToZero) << 6)      // immr
     778          20 :                                 | ((64 - NrBitsToZero - 1) << 0); // imms
     779             : 
     780          60 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
     781          20 :           .addReg(scratchSPReg, RegState::Kill)
     782          20 :           .addImm(andMaskEncoded);
     783             :       AFI->setStackRealigned(true);
     784             :     }
     785             :   }
     786             : 
     787             :   // If we need a base pointer, set it up here. It's whatever the value of the
     788             :   // stack pointer is at this point. Any variable size objects will be allocated
     789             :   // after this, so we can still use the base pointer to reference locals.
     790             :   //
     791             :   // FIXME: Clarify FrameSetup flags here.
     792             :   // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
     793             :   // needed.
     794        1365 :   if (RegInfo->hasBasePointer(MF)) {
     795          15 :     TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
     796          15 :                      false);
     797             :   }
     798             : 
     799        1365 :   if (needsFrameMoves) {
     800        1054 :     const DataLayout &TD = MF.getDataLayout();
     801        1054 :     const int StackGrowth = -TD.getPointerSize(0);
     802        1054 :     unsigned FramePtr = RegInfo->getFrameRegister(MF);
     803             :     // An example of the prologue:
     804             :     //
     805             :     //     .globl __foo
     806             :     //     .align 2
     807             :     //  __foo:
     808             :     // Ltmp0:
     809             :     //     .cfi_startproc
     810             :     //     .cfi_personality 155, ___gxx_personality_v0
     811             :     // Leh_func_begin:
     812             :     //     .cfi_lsda 16, Lexception33
     813             :     //
     814             :     //     stp  xa,bx, [sp, -#offset]!
     815             :     //     ...
     816             :     //     stp  x28, x27, [sp, #offset-32]
     817             :     //     stp  fp, lr, [sp, #offset-16]
     818             :     //     add  fp, sp, #offset - 16
     819             :     //     sub  sp, sp, #1360
     820             :     //
     821             :     // The Stack:
     822             :     //       +-------------------------------------------+
     823             :     // 10000 | ........ | ........ | ........ | ........ |
     824             :     // 10004 | ........ | ........ | ........ | ........ |
     825             :     //       +-------------------------------------------+
     826             :     // 10008 | ........ | ........ | ........ | ........ |
     827             :     // 1000c | ........ | ........ | ........ | ........ |
     828             :     //       +===========================================+
     829             :     // 10010 |                X28 Register               |
     830             :     // 10014 |                X28 Register               |
     831             :     //       +-------------------------------------------+
     832             :     // 10018 |                X27 Register               |
     833             :     // 1001c |                X27 Register               |
     834             :     //       +===========================================+
     835             :     // 10020 |                Frame Pointer              |
     836             :     // 10024 |                Frame Pointer              |
     837             :     //       +-------------------------------------------+
     838             :     // 10028 |                Link Register              |
     839             :     // 1002c |                Link Register              |
     840             :     //       +===========================================+
     841             :     // 10030 | ........ | ........ | ........ | ........ |
     842             :     // 10034 | ........ | ........ | ........ | ........ |
     843             :     //       +-------------------------------------------+
     844             :     // 10038 | ........ | ........ | ........ | ........ |
     845             :     // 1003c | ........ | ........ | ........ | ........ |
     846             :     //       +-------------------------------------------+
     847             :     //
     848             :     //     [sp] = 10030        ::    >>initial value<<
     849             :     //     sp = 10020          ::  stp fp, lr, [sp, #-16]!
     850             :     //     fp = sp == 10020    ::  mov fp, sp
     851             :     //     [sp] == 10020       ::  stp x28, x27, [sp, #-16]!
     852             :     //     sp == 10010         ::    >>final value<<
     853             :     //
     854             :     // The frame pointer (w29) points to address 10020. If we use an offset of
     855             :     // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
     856             :     // for w27, and -32 for w28:
     857             :     //
     858             :     //  Ltmp1:
     859             :     //     .cfi_def_cfa w29, 16
     860             :     //  Ltmp2:
     861             :     //     .cfi_offset w30, -8
     862             :     //  Ltmp3:
     863             :     //     .cfi_offset w29, -16
     864             :     //  Ltmp4:
     865             :     //     .cfi_offset w27, -24
     866             :     //  Ltmp5:
     867             :     //     .cfi_offset w28, -32
     868             : 
     869        1054 :     if (HasFP) {
     870             :       // Define the current CFA rule to use the provided FP.
     871         186 :       unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
     872         372 :       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
     873         186 :           nullptr, Reg, 2 * StackGrowth - FixedObject));
     874         372 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     875             :           .addCFIIndex(CFIIndex)
     876             :           .setMIFlags(MachineInstr::FrameSetup);
     877             :     } else {
     878             :       // Encode the stack size of the leaf function.
     879             :       unsigned CFIIndex = MF.addFrameInst(
     880         868 :           MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize()));
     881        1736 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     882             :           .addCFIIndex(CFIIndex)
     883             :           .setMIFlags(MachineInstr::FrameSetup);
     884             :     }
     885             : 
     886             :     // Now emit the moves for whatever callee saved regs we have (including FP,
     887             :     // LR if those are saved).
     888        1054 :     emitCalleeSavedFrameMoves(MBB, MBBI);
     889             :   }
     890             : }
     891             : 
     892       15022 : static void InsertReturnAddressAuth(MachineFunction &MF,
     893             :                                     MachineBasicBlock &MBB) {
     894       15022 :   if (!ShouldSignReturnAddress(MF))
     895       15016 :     return;
     896           6 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     897             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     898             : 
     899           6 :   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
     900           6 :   DebugLoc DL;
     901           6 :   if (MBBI != MBB.end())
     902             :     DL = MBBI->getDebugLoc();
     903             : 
     904             :   // The AUTIASP instruction assembles to a hint instruction before v8.3a so
     905             :   // this instruction can safely used for any v8a architecture.
     906             :   // From v8.3a onwards there are optimised authenticate LR and return
     907             :   // instructions, namely RETA{A,B}, that can be used instead.
     908           6 :   if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
     909           1 :       MBBI->getOpcode() == AArch64::RET_ReallyLR) {
     910           2 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::RETAA)).copyImplicitOps(*MBBI);
     911           1 :     MBB.erase(MBBI);
     912             :   } else {
     913          10 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::AUTIASP))
     914             :         .setMIFlag(MachineInstr::FrameDestroy);
     915             :   }
     916             : }
     917             : 
     918       15028 : void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
     919             :                                         MachineBasicBlock &MBB) const {
     920       15028 :   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
     921       15028 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     922       15028 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     923             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     924       15028 :   DebugLoc DL;
     925             :   bool IsTailCallReturn = false;
     926       15028 :   if (MBB.end() != MBBI) {
     927             :     DL = MBBI->getDebugLoc();
     928       15028 :     unsigned RetOpcode = MBBI->getOpcode();
     929       30056 :     IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
     930             :                        RetOpcode == AArch64::TCRETURNri ||
     931       15028 :                        RetOpcode == AArch64::TCRETURNriBTI;
     932             :   }
     933       15028 :   int NumBytes = MFI.getStackSize();
     934       15028 :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     935             : 
     936             :   // All calls are tail calls in GHC calling conv, and functions have no
     937             :   // prologue/epilogue.
     938       30056 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     939             :     return;
     940             : 
     941             :   // Initial and residual are named for consistency with the prologue. Note that
     942             :   // in the epilogue, the residual adjustment is executed first.
     943             :   uint64_t ArgumentPopSize = 0;
     944       15022 :   if (IsTailCallReturn) {
     945         225 :     MachineOperand &StackAdjust = MBBI->getOperand(1);
     946             : 
     947             :     // For a tail-call in a callee-pops-arguments environment, some or all of
     948             :     // the stack may actually be in use for the call's arguments, this is
     949             :     // calculated during LowerCall and consumed here...
     950         225 :     ArgumentPopSize = StackAdjust.getImm();
     951             :   } else {
     952             :     // ... otherwise the amount to pop is *all* of the argument space,
     953             :     // conveniently stored in the MachineFunctionInfo by
     954             :     // LowerFormalArguments. This will, of course, be zero for the C calling
     955             :     // convention.
     956       14797 :     ArgumentPopSize = AFI->getArgumentStackToRestore();
     957             :   }
     958             : 
     959             :   // The stack frame should be like below,
     960             :   //
     961             :   //      ----------------------                     ---
     962             :   //      |                    |                      |
     963             :   //      | BytesInStackArgArea|              CalleeArgStackSize
     964             :   //      | (NumReusableBytes) |                (of tail call)
     965             :   //      |                    |                     ---
     966             :   //      |                    |                      |
     967             :   //      ---------------------|        ---           |
     968             :   //      |                    |         |            |
     969             :   //      |   CalleeSavedReg   |         |            |
     970             :   //      | (CalleeSavedStackSize)|      |            |
     971             :   //      |                    |         |            |
     972             :   //      ---------------------|         |         NumBytes
     973             :   //      |                    |     StackSize  (StackAdjustUp)
     974             :   //      |   LocalStackSize   |         |            |
     975             :   //      | (covering callee   |         |            |
     976             :   //      |       args)        |         |            |
     977             :   //      |                    |         |            |
     978             :   //      ----------------------        ---          ---
     979             :   //
     980             :   // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
     981             :   //             = StackSize + ArgumentPopSize
     982             :   //
     983             :   // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
     984             :   // it as the 2nd argument of AArch64ISD::TC_RETURN.
     985             : 
     986       15022 :   auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
     987             : 
     988             :   bool IsWin64 =
     989             :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
     990       14934 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
     991             : 
     992             :   uint64_t AfterCSRPopSize = ArgumentPopSize;
     993       15022 :   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
     994       15022 :   bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
     995             :   // Assume we can't combine the last pop with the sp restore.
     996             : 
     997       15022 :   if (!CombineSPBump && PrologueSaveSize != 0) {
     998         853 :     MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
     999             :     // Converting the last ldp to a post-index ldp is valid only if the last
    1000             :     // ldp's offset is 0.
    1001         853 :     const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
    1002             :     // If the offset is 0, convert it to a post-index ldp.
    1003         853 :     if (OffsetOp.getImm() == 0) {
    1004             :       convertCalleeSaveRestoreToSPPrePostIncDec(MBB, Pop, DL, TII,
    1005         847 :                                                 PrologueSaveSize);
    1006             :     } else {
    1007             :       // If not, make sure to emit an add after the last ldp.
    1008             :       // We're doing this by transfering the size to be restored from the
    1009             :       // adjustment *before* the CSR pops to the adjustment *after* the CSR
    1010             :       // pops.
    1011           6 :       AfterCSRPopSize += PrologueSaveSize;
    1012             :     }
    1013             :   }
    1014             : 
    1015             :   // Move past the restores of the callee-saved registers.
    1016             :   // If we plan on combining the sp bump of the local stack size and the callee
    1017             :   // save stack size, we might need to adjust the CSR save and restore offsets.
    1018       15022 :   MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
    1019             :   MachineBasicBlock::iterator Begin = MBB.begin();
    1020       17404 :   while (LastPopI != Begin) {
    1021             :     --LastPopI;
    1022       16875 :     if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
    1023             :       ++LastPopI;
    1024             :       break;
    1025        2382 :     } else if (CombineSPBump)
    1026        1016 :       fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize());
    1027             :   }
    1028             : 
    1029             :   // If there is a single SP update, insert it before the ret and we're done.
    1030       15022 :   if (CombineSPBump) {
    1031         746 :     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
    1032         746 :                     NumBytes + AfterCSRPopSize, TII,
    1033             :                     MachineInstr::FrameDestroy);
    1034         746 :     return;
    1035             :   }
    1036             : 
    1037       14276 :   NumBytes -= PrologueSaveSize;
    1038             :   assert(NumBytes >= 0 && "Negative stack allocation size!?");
    1039             : 
    1040       14276 :   if (!hasFP(MF)) {
    1041       14089 :     bool RedZone = canUseRedZone(MF);
    1042             :     // If this was a redzone leaf function, we don't need to restore the
    1043             :     // stack pointer (but we may need to pop stack args for fastcc).
    1044       14089 :     if (RedZone && AfterCSRPopSize == 0)
    1045             :       return;
    1046             : 
    1047       14036 :     bool NoCalleeSaveRestore = PrologueSaveSize == 0;
    1048       14036 :     int StackRestoreBytes = RedZone ? 0 : NumBytes;
    1049       14036 :     if (NoCalleeSaveRestore)
    1050       13370 :       StackRestoreBytes += AfterCSRPopSize;
    1051             : 
    1052             :     // If we were able to combine the local stack pop with the argument pop,
    1053             :     // then we're done.
    1054       14036 :     bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
    1055             : 
    1056             :     // If we're done after this, make sure to help the load store optimizer.
    1057       14036 :     if (Done)
    1058       28054 :       adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
    1059             : 
    1060       14036 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
    1061             :                     StackRestoreBytes, TII, MachineInstr::FrameDestroy);
    1062       14036 :     if (Done)
    1063             :       return;
    1064             : 
    1065             :     NumBytes = 0;
    1066             :   }
    1067             : 
    1068             :   // Restore the original stack pointer.
    1069             :   // FIXME: Rather than doing the math here, we should instead just use
    1070             :   // non-post-indexed loads for the restores if we aren't actually going to
    1071             :   // be able to save any instructions.
    1072         196 :   if (MFI.hasVarSizedObjects() || AFI->isStackRealigned())
    1073          54 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
    1074          54 :                     -AFI->getCalleeSavedStackSize() + 16, TII,
    1075             :                     MachineInstr::FrameDestroy);
    1076         142 :   else if (NumBytes)
    1077           7 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
    1078             :                     MachineInstr::FrameDestroy);
    1079             : 
    1080             :   // This must be placed after the callee-save restore code because that code
    1081             :   // assumes the SP is at the same location as it was after the callee-save save
    1082             :   // code in the prologue.
    1083         196 :   if (AfterCSRPopSize) {
    1084             :     // Find an insertion point for the first ldp so that it goes before the
    1085             :     // shadow call stack epilog instruction. This ensures that the restore of
    1086             :     // lr from x18 is placed after the restore from sp.
    1087           9 :     auto FirstSPPopI = MBB.getFirstTerminator();
    1088          11 :     while (FirstSPPopI != Begin) {
    1089          11 :       auto Prev = std::prev(FirstSPPopI);
    1090          22 :       if (Prev->getOpcode() != AArch64::LDRXpre ||
    1091           2 :           Prev->getOperand(0).getReg() == AArch64::SP)
    1092             :         break;
    1093             :       FirstSPPopI = Prev;
    1094             :     }
    1095             : 
    1096           9 :     adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
    1097             : 
    1098           9 :     emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
    1099             :                     AfterCSRPopSize, TII, MachineInstr::FrameDestroy);
    1100             :   }
    1101             : }
    1102             : 
    1103             : /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
    1104             : /// debug info.  It's the same as what we use for resolving the code-gen
    1105             : /// references for now.  FIXME: This can go wrong when references are
    1106             : /// SP-relative and simple call frames aren't used.
    1107          15 : int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
    1108             :                                                  int FI,
    1109             :                                                  unsigned &FrameReg) const {
    1110          15 :   return resolveFrameIndexReference(MF, FI, FrameReg);
    1111             : }
    1112             : 
    1113        3947 : int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
    1114             :                                                      int FI, unsigned &FrameReg,
    1115             :                                                      bool PreferFP) const {
    1116        3947 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
    1117             :   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
    1118        3947 :       MF.getSubtarget().getRegisterInfo());
    1119             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1120        3947 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
    1121             :   bool IsWin64 =
    1122        3947 :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
    1123        3555 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
    1124        3947 :   int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16;
    1125        3947 :   int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
    1126             :   bool isFixed = MFI.isFixedObjectIndex(FI);
    1127        3575 :   bool isCSR = !isFixed && MFI.getObjectOffset(FI) >=
    1128        3575 :                                -((int)AFI->getCalleeSavedStackSize());
    1129             : 
    1130             :   // Use frame pointer to reference fixed objects. Use it for locals if
    1131             :   // there are VLAs or a dynamically realigned SP (and thus the SP isn't
    1132             :   // reliable as a base). Make sure useFPForScavengingIndex() does the
    1133             :   // right thing for the emergency spill slot.
    1134             :   bool UseFP = false;
    1135        3947 :   if (AFI->hasStackFrame()) {
    1136             :     // Note: Keeping the following as multiple 'if' statements rather than
    1137             :     // merging to a single expression for readability.
    1138             :     //
    1139             :     // Argument access should always use the FP.
    1140        3011 :     if (isFixed) {
    1141         194 :       UseFP = hasFP(MF);
    1142        2817 :     } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
    1143             :       // References to the CSR area must use FP if we're re-aligning the stack
    1144             :       // since the dynamically-sized alignment padding is between the SP/BP and
    1145             :       // the CSR area.
    1146             :       assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
    1147             :       UseFP = true;
    1148        2816 :     } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
    1149             :       // If the FPOffset is negative, we have to keep in mind that the
    1150             :       // available offset range for negative offsets is smaller than for
    1151             :       // positive ones. If an offset is
    1152             :       // available via the FP and the SP, use whichever is closest.
    1153         476 :       bool FPOffsetFits = FPOffset >= -256;
    1154         476 :       PreferFP |= Offset > -FPOffset;
    1155             : 
    1156         476 :       if (MFI.hasVarSizedObjects()) {
    1157             :         // If we have variable sized objects, we can use either FP or BP, as the
    1158             :         // SP offset is unknown. We can use the base pointer if we have one and
    1159             :         // FP is not preferred. If not, we're stuck with using FP.
    1160          45 :         bool CanUseBP = RegInfo->hasBasePointer(MF);
    1161          45 :         if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
    1162             :           UseFP = PreferFP;
    1163          40 :         else if (!CanUseBP) // Can't use BP. Forced to use FP.
    1164             :           UseFP = true;
    1165             :         // else we can use BP and FP, but the offset from FP won't fit.
    1166             :         // That will make us scavenge registers which we can probably avoid by
    1167             :         // using BP. If it won't fit for BP either, we'll scavenge anyway.
    1168         431 :       } else if (FPOffset >= 0) {
    1169             :         // Use SP or FP, whichever gives us the best chance of the offset
    1170             :         // being in range for direct access. If the FPOffset is positive,
    1171             :         // that'll always be best, as the SP will be even further away.
    1172             :         UseFP = true;
    1173             :       } else {
    1174             :         // We have the choice between FP and (SP or BP).
    1175         431 :         if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
    1176             :           UseFP = true;
    1177             :       }
    1178             :     }
    1179             :   }
    1180             : 
    1181             :   assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
    1182             :          "In the presence of dynamic stack pointer realignment, "
    1183             :          "non-argument/CSR objects cannot be accessed through the frame pointer");
    1184             : 
    1185         199 :   if (UseFP) {
    1186         355 :     FrameReg = RegInfo->getFrameRegister(MF);
    1187         355 :     return FPOffset;
    1188             :   }
    1189             : 
    1190             :   // Use the base pointer if we have one.
    1191        3592 :   if (RegInfo->hasBasePointer(MF))
    1192          11 :     FrameReg = RegInfo->getBaseRegister();
    1193             :   else {
    1194             :     assert(!MFI.hasVarSizedObjects() &&
    1195             :            "Can't use SP when we have var sized objects.");
    1196        3581 :     FrameReg = AArch64::SP;
    1197             :     // If we're using the red zone for this function, the SP won't actually
    1198             :     // be adjusted, so the offsets will be negative. They're also all
    1199             :     // within range of the signed 9-bit immediate instructions.
    1200        3581 :     if (canUseRedZone(MF))
    1201          23 :       Offset -= AFI->getLocalStackSize();
    1202             :   }
    1203             : 
    1204             :   return Offset;
    1205             : }
    1206             : 
    1207             : static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
    1208             :   // Do not set a kill flag on values that are also marked as live-in. This
    1209             :   // happens with the @llvm-returnaddress intrinsic and with arguments passed in
    1210             :   // callee saved registers.
    1211             :   // Omitting the kill flags is conservatively correct even if the live-in
    1212             :   // is not used after all.
    1213        4033 :   bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
    1214             :   return getKillRegState(!IsLiveIn);
    1215             : }
    1216             : 
    1217        3967 : static bool produceCompactUnwindFrame(MachineFunction &MF) {
    1218        3967 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
    1219        3967 :   AttributeList Attrs = MF.getFunction().getAttributes();
    1220        5620 :   return Subtarget.isTargetMachO() &&
    1221        3306 :          !(Subtarget.getTargetLowering()->supportSwiftError() &&
    1222        1653 :            Attrs.hasAttrSomewhere(Attribute::SwiftError));
    1223             : }
    1224             : 
    1225             : namespace {
    1226             : 
    1227             : struct RegPairInfo {
    1228             :   unsigned Reg1 = AArch64::NoRegister;
    1229             :   unsigned Reg2 = AArch64::NoRegister;
    1230             :   int FrameIdx;
    1231             :   int Offset;
    1232             :   enum RegType { GPR, FPR64, FPR128 } Type;
    1233             : 
    1234           0 :   RegPairInfo() = default;
    1235             : 
    1236           0 :   bool isPaired() const { return Reg2 != AArch64::NoRegister; }
    1237             : };
    1238             : 
    1239             : } // end anonymous namespace
    1240             : 
    1241           0 : static void computeCalleeSaveRegisterPairs(
    1242             :     MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
    1243             :     const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
    1244             :     bool &NeedShadowCallStackProlog) {
    1245             : 
    1246           0 :   if (CSI.empty())
    1247           0 :     return;
    1248             : 
    1249           0 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1250           0 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1251             :   CallingConv::ID CC = MF.getFunction().getCallingConv();
    1252           0 :   unsigned Count = CSI.size();
    1253             :   (void)CC;
    1254             :   // MachO's compact unwind format relies on all registers being stored in
    1255             :   // pairs.
    1256             :   assert((!produceCompactUnwindFrame(MF) ||
    1257             :           CC == CallingConv::PreserveMost ||
    1258             :           (Count & 1) == 0) &&
    1259             :          "Odd number of callee-saved regs to spill!");
    1260           0 :   int Offset = AFI->getCalleeSavedStackSize();
    1261             : 
    1262           0 :   for (unsigned i = 0; i < Count; ++i) {
    1263             :     RegPairInfo RPI;
    1264           0 :     RPI.Reg1 = CSI[i].getReg();
    1265             : 
    1266           0 :     if (AArch64::GPR64RegClass.contains(RPI.Reg1))
    1267           0 :       RPI.Type = RegPairInfo::GPR;
    1268           0 :     else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
    1269           0 :       RPI.Type = RegPairInfo::FPR64;
    1270           0 :     else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
    1271           0 :       RPI.Type = RegPairInfo::FPR128;
    1272             :     else
    1273           0 :       llvm_unreachable("Unsupported register class.");
    1274             : 
    1275             :     // Add the next reg to the pair if it is in the same register class.
    1276           0 :     if (i + 1 < Count) {
    1277           0 :       unsigned NextReg = CSI[i + 1].getReg();
    1278           0 :       switch (RPI.Type) {
    1279           0 :       case RegPairInfo::GPR:
    1280           0 :         if (AArch64::GPR64RegClass.contains(NextReg))
    1281           0 :           RPI.Reg2 = NextReg;
    1282             :         break;
    1283           0 :       case RegPairInfo::FPR64:
    1284           0 :         if (AArch64::FPR64RegClass.contains(NextReg))
    1285           0 :           RPI.Reg2 = NextReg;
    1286             :         break;
    1287           0 :       case RegPairInfo::FPR128:
    1288           0 :         if (AArch64::FPR128RegClass.contains(NextReg))
    1289           0 :           RPI.Reg2 = NextReg;
    1290             :         break;
    1291             :       }
    1292             :     }
    1293             : 
    1294             :     // If either of the registers to be saved is the lr register, it means that
    1295             :     // we also need to save lr in the shadow call stack.
    1296           0 :     if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
    1297           0 :         MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
    1298           0 :       if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
    1299           0 :         report_fatal_error("Must reserve x18 to use shadow call stack");
    1300           0 :       NeedShadowCallStackProlog = true;
    1301             :     }
    1302             : 
    1303             :     // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
    1304             :     // list to come in sorted by frame index so that we can issue the store
    1305             :     // pair instructions directly. Assert if we see anything otherwise.
    1306             :     //
    1307             :     // The order of the registers in the list is controlled by
    1308             :     // getCalleeSavedRegs(), so they will always be in-order, as well.
    1309             :     assert((!RPI.isPaired() ||
    1310             :             (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
    1311             :            "Out of order callee saved regs!");
    1312             : 
    1313             :     // MachO's compact unwind format relies on all registers being stored in
    1314             :     // adjacent register pairs.
    1315             :     assert((!produceCompactUnwindFrame(MF) ||
    1316             :             CC == CallingConv::PreserveMost ||
    1317             :             (RPI.isPaired() &&
    1318             :              ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
    1319             :               RPI.Reg1 + 1 == RPI.Reg2))) &&
    1320             :            "Callee-save registers not saved as adjacent register pair!");
    1321             : 
    1322           0 :     RPI.FrameIdx = CSI[i].getFrameIdx();
    1323             : 
    1324           0 :     int Scale = RPI.Type == RegPairInfo::FPR128 ? 16 : 8;
    1325           0 :     Offset -= RPI.isPaired() ? 2 * Scale : Scale;
    1326             : 
    1327             :     // Round up size of non-pair to pair size if we need to pad the
    1328             :     // callee-save area to ensure 16-byte alignment.
    1329           0 :     if (AFI->hasCalleeSaveStackFreeSpace() &&
    1330           0 :         RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired()) {
    1331           0 :       Offset -= 8;
    1332             :       assert(Offset % 16 == 0);
    1333             :       assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
    1334             :       MFI.setObjectAlignment(RPI.FrameIdx, 16);
    1335             :     }
    1336             : 
    1337             :     assert(Offset % Scale == 0);
    1338           0 :     RPI.Offset = Offset / Scale;
    1339             :     assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
    1340             :            "Offset out of bounds for LDP/STP immediate");
    1341             : 
    1342           0 :     RegPairs.push_back(RPI);
    1343           0 :     if (RPI.isPaired())
    1344             :       ++i;
    1345             :   }
    1346             : }
    1347             : 
    1348        1365 : bool AArch64FrameLowering::spillCalleeSavedRegisters(
    1349             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
    1350             :     const std::vector<CalleeSavedInfo> &CSI,
    1351             :     const TargetRegisterInfo *TRI) const {
    1352        1365 :   MachineFunction &MF = *MBB.getParent();
    1353        1365 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1354        1365 :   DebugLoc DL;
    1355        1365 :   SmallVector<RegPairInfo, 8> RegPairs;
    1356             : 
    1357        1365 :   bool NeedShadowCallStackProlog = false;
    1358        1365 :   computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
    1359             :                                  NeedShadowCallStackProlog);
    1360        1365 :   const MachineRegisterInfo &MRI = MF.getRegInfo();
    1361             : 
    1362        1365 :   if (NeedShadowCallStackProlog) {
    1363             :     // Shadow call stack prolog: str x30, [x18], #8
    1364           6 :     BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
    1365           2 :         .addReg(AArch64::X18, RegState::Define)
    1366           2 :         .addReg(AArch64::LR)
    1367           2 :         .addReg(AArch64::X18)
    1368             :         .addImm(8)
    1369             :         .setMIFlag(MachineInstr::FrameSetup);
    1370             : 
    1371             :     // This instruction also makes x18 live-in to the entry block.
    1372             :     MBB.addLiveIn(AArch64::X18);
    1373             :   }
    1374             : 
    1375        3766 :   for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
    1376             :        ++RPII) {
    1377        2401 :     RegPairInfo RPI = *RPII;
    1378             :     unsigned Reg1 = RPI.Reg1;
    1379             :     unsigned Reg2 = RPI.Reg2;
    1380             :     unsigned StrOpc;
    1381             : 
    1382             :     // Issue sequence of spills for cs regs.  The first spill may be converted
    1383             :     // to a pre-decrement store later by emitPrologue if the callee-save stack
    1384             :     // area allocation can't be combined with the local stack area allocation.
    1385             :     // For example:
    1386             :     //    stp     x22, x21, [sp, #0]     // addImm(+0)
    1387             :     //    stp     x20, x19, [sp, #16]    // addImm(+2)
    1388             :     //    stp     fp, lr, [sp, #32]      // addImm(+4)
    1389             :     // Rationale: This sequence saves uop updates compared to a sequence of
    1390             :     // pre-increment spills like stp xi,xj,[sp,#-16]!
    1391             :     // Note: Similar rationale and sequence for restores in epilog.
    1392             :     unsigned Size, Align;
    1393        2401 :     switch (RPI.Type) {
    1394        2103 :     case RegPairInfo::GPR:
    1395        2103 :        StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
    1396             :        Size = 8;
    1397             :        Align = 8;
    1398             :        break;
    1399         279 :     case RegPairInfo::FPR64:
    1400         279 :        StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
    1401             :        Size = 8;
    1402             :        Align = 8;
    1403             :        break;
    1404          19 :     case RegPairInfo::FPR128:
    1405          19 :        StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
    1406             :        Size = 16;
    1407             :        Align = 16;
    1408             :        break;
    1409             :     }
    1410             :     LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
    1411             :                if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
    1412             :                dbgs() << ") -> fi#(" << RPI.FrameIdx;
    1413             :                if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
    1414             :                dbgs() << ")\n");
    1415             : 
    1416        4802 :     MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
    1417        2401 :     if (!MRI.isReserved(Reg1))
    1418        2386 :       MBB.addLiveIn(Reg1);
    1419        2401 :     if (RPI.isPaired()) {
    1420        1632 :       if (!MRI.isReserved(Reg2))
    1421        1153 :         MBB.addLiveIn(Reg2);
    1422        1632 :       MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
    1423             :       MIB.addMemOperand(MF.getMachineMemOperand(
    1424             :           MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
    1425        3264 :           MachineMemOperand::MOStore, Size, Align));
    1426             :     }
    1427        2401 :     MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
    1428        2401 :         .addReg(AArch64::SP)
    1429        2401 :         .addImm(RPI.Offset) // [sp, #offset*scale],
    1430             :                             // where factor*scale is implicit
    1431             :         .setMIFlag(MachineInstr::FrameSetup);
    1432             :     MIB.addMemOperand(MF.getMachineMemOperand(
    1433             :         MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
    1434        4802 :         MachineMemOperand::MOStore, Size, Align));
    1435             :   }
    1436        1365 :   return true;
    1437             : }
    1438             : 
    1439        1372 : bool AArch64FrameLowering::restoreCalleeSavedRegisters(
    1440             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
    1441             :     std::vector<CalleeSavedInfo> &CSI,
    1442             :     const TargetRegisterInfo *TRI) const {
    1443        1372 :   MachineFunction &MF = *MBB.getParent();
    1444        1372 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1445        1372 :   DebugLoc DL;
    1446        1372 :   SmallVector<RegPairInfo, 8> RegPairs;
    1447             : 
    1448        1372 :   if (MI != MBB.end())
    1449             :     DL = MI->getDebugLoc();
    1450             : 
    1451        1372 :   bool NeedShadowCallStackProlog = false;
    1452        1372 :   computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
    1453             :                                  NeedShadowCallStackProlog);
    1454             : 
    1455             :   auto EmitMI = [&](const RegPairInfo &RPI) {
    1456             :     unsigned Reg1 = RPI.Reg1;
    1457             :     unsigned Reg2 = RPI.Reg2;
    1458             : 
    1459             :     // Issue sequence of restores for cs regs. The last restore may be converted
    1460             :     // to a post-increment load later by emitEpilogue if the callee-save stack
    1461             :     // area allocation can't be combined with the local stack area allocation.
    1462             :     // For example:
    1463             :     //    ldp     fp, lr, [sp, #32]       // addImm(+4)
    1464             :     //    ldp     x20, x19, [sp, #16]     // addImm(+2)
    1465             :     //    ldp     x22, x21, [sp, #0]      // addImm(+0)
    1466             :     // Note: see comment in spillCalleeSavedRegisters()
    1467             :     unsigned LdrOpc;
    1468             :     unsigned Size, Align;
    1469             :     switch (RPI.Type) {
    1470             :     case RegPairInfo::GPR:
    1471             :        LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
    1472             :        Size = 8;
    1473             :        Align = 8;
    1474             :        break;
    1475             :     case RegPairInfo::FPR64:
    1476             :        LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
    1477             :        Size = 8;
    1478             :        Align = 8;
    1479             :        break;
    1480             :     case RegPairInfo::FPR128:
    1481             :        LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
    1482             :        Size = 16;
    1483             :        Align = 16;
    1484             :        break;
    1485             :     }
    1486             :     LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
    1487             :                if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
    1488             :                dbgs() << ") -> fi#(" << RPI.FrameIdx;
    1489             :                if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
    1490             :                dbgs() << ")\n");
    1491             : 
    1492             :     MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
    1493             :     if (RPI.isPaired()) {
    1494             :       MIB.addReg(Reg2, getDefRegState(true));
    1495             :       MIB.addMemOperand(MF.getMachineMemOperand(
    1496             :           MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
    1497             :           MachineMemOperand::MOLoad, Size, Align));
    1498             :     }
    1499             :     MIB.addReg(Reg1, getDefRegState(true))
    1500             :         .addReg(AArch64::SP)
    1501             :         .addImm(RPI.Offset) // [sp, #offset*scale]
    1502             :                             // where factor*scale is implicit
    1503             :         .setMIFlag(MachineInstr::FrameDestroy);
    1504             :     MIB.addMemOperand(MF.getMachineMemOperand(
    1505             :         MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
    1506             :         MachineMemOperand::MOLoad, Size, Align));
    1507        1372 :   };
    1508             : 
    1509        1372 :   if (ReverseCSRRestoreSeq)
    1510          26 :     for (const RegPairInfo &RPI : reverse(RegPairs))
    1511          20 :       EmitMI(RPI);
    1512             :   else
    1513        3726 :     for (const RegPairInfo &RPI : RegPairs)
    1514        2360 :       EmitMI(RPI);
    1515             : 
    1516        1372 :   if (NeedShadowCallStackProlog) {
    1517             :     // Shadow call stack epilog: ldr x30, [x18, #-8]!
    1518           6 :     BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
    1519           2 :         .addReg(AArch64::X18, RegState::Define)
    1520           2 :         .addReg(AArch64::LR, RegState::Define)
    1521           2 :         .addReg(AArch64::X18)
    1522             :         .addImm(-8)
    1523             :         .setMIFlag(MachineInstr::FrameDestroy);
    1524             :   }
    1525             : 
    1526        1372 :   return true;
    1527             : }
    1528             : 
    1529       15053 : void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
    1530             :                                                 BitVector &SavedRegs,
    1531             :                                                 RegScavenger *RS) const {
    1532             :   // All calls are tail calls in GHC calling conv, and functions have no
    1533             :   // prologue/epilogue.
    1534       30106 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
    1535             :     return;
    1536             : 
    1537       15047 :   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
    1538             :   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
    1539       15047 :       MF.getSubtarget().getRegisterInfo());
    1540       15047 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1541             :   unsigned UnspilledCSGPR = AArch64::NoRegister;
    1542             :   unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
    1543             : 
    1544       15047 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1545       15047 :   const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
    1546             : 
    1547       15047 :   unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
    1548       15047 :                                 ? RegInfo->getBaseRegister()
    1549             :                                 : (unsigned)AArch64::NoRegister;
    1550             : 
    1551             :   unsigned ExtraCSSpill = 0;
    1552             :   // Figure out which callee-saved registers to save/restore.
    1553      316240 :   for (unsigned i = 0; CSRegs[i]; ++i) {
    1554      301193 :     const unsigned Reg = CSRegs[i];
    1555             : 
    1556             :     // Add the base pointer register to SavedRegs if it is callee-save.
    1557      301193 :     if (Reg == BasePointerReg)
    1558             :       SavedRegs.set(Reg);
    1559             : 
    1560             :     bool RegUsed = SavedRegs.test(Reg);
    1561      301193 :     unsigned PairedReg = CSRegs[i ^ 1];
    1562      301193 :     if (!RegUsed) {
    1563      474548 :       if (AArch64::GPR64RegClass.contains(Reg) &&
    1564      177284 :           !RegInfo->isReservedReg(MF, Reg)) {
    1565             :         UnspilledCSGPR = Reg;
    1566             :         UnspilledCSGPRPaired = PairedReg;
    1567             :       }
    1568      297264 :       continue;
    1569             :     }
    1570             : 
    1571             :     // MachO's compact unwind format relies on all registers being stored in
    1572             :     // pairs.
    1573             :     // FIXME: the usual format is actually better if unwinding isn't needed.
    1574        3929 :     if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
    1575             :         !SavedRegs.test(PairedReg)) {
    1576             :       SavedRegs.set(PairedReg);
    1577         769 :       if (AArch64::GPR64RegClass.contains(PairedReg) &&
    1578         377 :           !RegInfo->isReservedReg(MF, PairedReg))
    1579             :         ExtraCSSpill = PairedReg;
    1580             :     }
    1581             :   }
    1582             : 
    1583             :   // Calculates the callee saved stack size.
    1584             :   unsigned CSStackSize = 0;
    1585       15047 :   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1586       15047 :   const MachineRegisterInfo &MRI = MF.getRegInfo();
    1587       18996 :   for (unsigned Reg : SavedRegs.set_bits())
    1588        3949 :     CSStackSize += TRI->getRegSizeInBits(Reg, MRI) / 8;
    1589             : 
    1590             :   // Save number of saved regs, so we can easily update CSStackSize later.
    1591             :   unsigned NumSavedRegs = SavedRegs.count();
    1592             : 
    1593             :   // The frame record needs to be created by saving the appropriate registers
    1594       15047 :   unsigned EstimatedStackSize = MFI.estimateStackSize(MF);
    1595       29794 :   if (hasFP(MF) ||
    1596       14747 :       windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
    1597             :     SavedRegs.set(AArch64::FP);
    1598             :     SavedRegs.set(AArch64::LR);
    1599             :   }
    1600             : 
    1601             :   LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
    1602             :              for (unsigned Reg
    1603             :                   : SavedRegs.set_bits()) dbgs()
    1604             :              << ' ' << printReg(Reg, RegInfo);
    1605             :              dbgs() << "\n";);
    1606             : 
    1607             :   // If any callee-saved registers are used, the frame cannot be eliminated.
    1608             :   bool CanEliminateFrame = SavedRegs.count() == 0;
    1609             : 
    1610             :   // The CSR spill slots have not been allocated yet, so estimateStackSize
    1611             :   // won't include them.
    1612       15047 :   unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
    1613       15047 :   bool BigStack = (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
    1614       15047 :   if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
    1615             :     AFI->setHasStackFrame(true);
    1616             : 
    1617             :   // Estimate if we might need to scavenge a register at some point in order
    1618             :   // to materialize a stack offset. If so, either spill one additional
    1619             :   // callee-saved register or reserve a special spill slot to facilitate
    1620             :   // register scavenging. If we already spilled an extra callee-saved register
    1621             :   // above to keep the number of spills even, we don't need to do anything else
    1622             :   // here.
    1623       15047 :   if (BigStack) {
    1624          63 :     if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
    1625             :       LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
    1626             :                         << " to get a scratch register.\n");
    1627             :       SavedRegs.set(UnspilledCSGPR);
    1628             :       // MachO's compact unwind format relies on all registers being stored in
    1629             :       // pairs, so if we need to spill one extra for BigStack, then we need to
    1630             :       // store the pair.
    1631          38 :       if (produceCompactUnwindFrame(MF))
    1632             :         SavedRegs.set(UnspilledCSGPRPaired);
    1633             :       ExtraCSSpill = UnspilledCSGPRPaired;
    1634             :     }
    1635             : 
    1636             :     // If we didn't find an extra callee-saved register to spill, create
    1637             :     // an emergency spill slot.
    1638          63 :     if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
    1639          25 :       const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1640             :       const TargetRegisterClass &RC = AArch64::GPR64RegClass;
    1641             :       unsigned Size = TRI->getSpillSize(RC);
    1642             :       unsigned Align = TRI->getSpillAlignment(RC);
    1643          25 :       int FI = MFI.CreateStackObject(Size, Align, false);
    1644             :       RS->addScavengingFrameIndex(FI);
    1645             :       LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
    1646             :                         << " as the emergency spill slot.\n");
    1647             :     }
    1648             :   }
    1649             : 
    1650             :   // Adding the size of additional 64bit GPR saves.
    1651       15047 :   CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
    1652       30094 :   unsigned AlignedCSStackSize = alignTo(CSStackSize, 16);
    1653             :   LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
    1654             :                << EstimatedStackSize + AlignedCSStackSize
    1655             :                << " bytes.\n");
    1656             : 
    1657             :   // Round up to register pair alignment to avoid additional SP adjustment
    1658             :   // instructions.
    1659             :   AFI->setCalleeSavedStackSize(AlignedCSStackSize);
    1660       15047 :   AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
    1661             : }
    1662             : 
    1663         493 : bool AArch64FrameLowering::enableStackSlotScavenging(
    1664             :     const MachineFunction &MF) const {
    1665             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1666         493 :   return AFI->hasCalleeSaveStackFreeSpace();
    1667             : }

Generated by: LCOV version 1.13