LCOV - code coverage report
Current view: top level - lib/Target/AArch64 - AArch64FrameLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 438 495 88.5 %
Date: 2018-09-23 13:06:45 Functions: 23 26 88.5 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the AArch64 implementation of TargetFrameLowering class.
      11             : //
      12             : // On AArch64, stack frames are structured as follows:
      13             : //
      14             : // The stack grows downward.
      15             : //
      16             : // All of the individual frame areas on the frame below are optional, i.e. it's
      17             : // possible to create a function so that the particular area isn't present
      18             : // in the frame.
      19             : //
      20             : // At function entry, the "frame" looks as follows:
      21             : //
      22             : // |                                   | Higher address
      23             : // |-----------------------------------|
      24             : // |                                   |
      25             : // | arguments passed on the stack     |
      26             : // |                                   |
      27             : // |-----------------------------------| <- sp
      28             : // |                                   | Lower address
      29             : //
      30             : //
      31             : // After the prologue has run, the frame has the following general structure.
      32             : // Note that this doesn't depict the case where a red-zone is used. Also,
      33             : // technically the last frame area (VLAs) doesn't get created until in the
      34             : // main function body, after the prologue is run. However, it's depicted here
      35             : // for completeness.
      36             : //
      37             : // |                                   | Higher address
      38             : // |-----------------------------------|
      39             : // |                                   |
      40             : // | arguments passed on the stack     |
      41             : // |                                   |
      42             : // |-----------------------------------|
      43             : // |                                   |
      44             : // | (Win64 only) varargs from reg     |
      45             : // |                                   |
      46             : // |-----------------------------------|
      47             : // |                                   |
      48             : // | prev_fp, prev_lr                  |
      49             : // | (a.k.a. "frame record")           |
      50             : // |-----------------------------------| <- fp(=x29)
      51             : // |                                   |
      52             : // | other callee-saved registers      |
      53             : // |                                   |
      54             : // |-----------------------------------|
      55             : // |.empty.space.to.make.part.below....|
      56             : // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
      57             : // |.the.standard.16-byte.alignment....|  compile time; if present)
      58             : // |-----------------------------------|
      59             : // |                                   |
      60             : // | local variables of fixed size     |
      61             : // | including spill slots             |
      62             : // |-----------------------------------| <- bp(not defined by ABI,
      63             : // |.variable-sized.local.variables....|       LLVM chooses X19)
      64             : // |.(VLAs)............................| (size of this area is unknown at
      65             : // |...................................|  compile time)
      66             : // |-----------------------------------| <- sp
      67             : // |                                   | Lower address
      68             : //
      69             : //
      70             : // To access the data in a frame, at-compile time, a constant offset must be
      71             : // computable from one of the pointers (fp, bp, sp) to access it. The size
      72             : // of the areas with a dotted background cannot be computed at compile-time
      73             : // if they are present, making it required to have all three of fp, bp and
      74             : // sp to be set up to be able to access all contents in the frame areas,
      75             : // assuming all of the frame areas are non-empty.
      76             : //
      77             : // For most functions, some of the frame areas are empty. For those functions,
      78             : // it may not be necessary to set up fp or bp:
      79             : // * A base pointer is definitely needed when there are both VLAs and local
      80             : //   variables with more-than-default alignment requirements.
      81             : // * A frame pointer is definitely needed when there are local variables with
      82             : //   more-than-default alignment requirements.
      83             : //
      84             : // In some cases when a base pointer is not strictly needed, it is generated
      85             : // anyway when offsets from the frame pointer to access local variables become
      86             : // so large that the offset can't be encoded in the immediate fields of loads
      87             : // or stores.
      88             : //
      89             : // FIXME: also explain the redzone concept.
      90             : // FIXME: also explain the concept of reserved call frames.
      91             : //
      92             : //===----------------------------------------------------------------------===//
      93             : 
      94             : #include "AArch64FrameLowering.h"
      95             : #include "AArch64InstrInfo.h"
      96             : #include "AArch64MachineFunctionInfo.h"
      97             : #include "AArch64RegisterInfo.h"
      98             : #include "AArch64Subtarget.h"
      99             : #include "AArch64TargetMachine.h"
     100             : #include "MCTargetDesc/AArch64AddressingModes.h"
     101             : #include "llvm/ADT/ScopeExit.h"
     102             : #include "llvm/ADT/SmallVector.h"
     103             : #include "llvm/ADT/Statistic.h"
     104             : #include "llvm/CodeGen/LivePhysRegs.h"
     105             : #include "llvm/CodeGen/MachineBasicBlock.h"
     106             : #include "llvm/CodeGen/MachineFrameInfo.h"
     107             : #include "llvm/CodeGen/MachineFunction.h"
     108             : #include "llvm/CodeGen/MachineInstr.h"
     109             : #include "llvm/CodeGen/MachineInstrBuilder.h"
     110             : #include "llvm/CodeGen/MachineMemOperand.h"
     111             : #include "llvm/CodeGen/MachineModuleInfo.h"
     112             : #include "llvm/CodeGen/MachineOperand.h"
     113             : #include "llvm/CodeGen/MachineRegisterInfo.h"
     114             : #include "llvm/CodeGen/RegisterScavenging.h"
     115             : #include "llvm/CodeGen/TargetInstrInfo.h"
     116             : #include "llvm/CodeGen/TargetRegisterInfo.h"
     117             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
     118             : #include "llvm/IR/Attributes.h"
     119             : #include "llvm/IR/CallingConv.h"
     120             : #include "llvm/IR/DataLayout.h"
     121             : #include "llvm/IR/DebugLoc.h"
     122             : #include "llvm/IR/Function.h"
     123             : #include "llvm/MC/MCDwarf.h"
     124             : #include "llvm/Support/CommandLine.h"
     125             : #include "llvm/Support/Debug.h"
     126             : #include "llvm/Support/ErrorHandling.h"
     127             : #include "llvm/Support/MathExtras.h"
     128             : #include "llvm/Support/raw_ostream.h"
     129             : #include "llvm/Target/TargetMachine.h"
     130             : #include "llvm/Target/TargetOptions.h"
     131             : #include <cassert>
     132             : #include <cstdint>
     133             : #include <iterator>
     134             : #include <vector>
     135             : 
     136             : using namespace llvm;
     137             : 
     138             : #define DEBUG_TYPE "frame-info"
     139             : 
     140             : static cl::opt<bool> EnableRedZone("aarch64-redzone",
     141             :                                    cl::desc("enable use of redzone on AArch64"),
     142             :                                    cl::init(false), cl::Hidden);
     143             : 
     144             : static cl::opt<bool>
     145             :     ReverseCSRRestoreSeq("reverse-csr-restore-seq",
     146             :                          cl::desc("reverse the CSR restore sequence"),
     147             :                          cl::init(false), cl::Hidden);
     148             : 
     149             : STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
     150             : 
     151             : /// This is the biggest offset to the stack pointer we can encode in aarch64
     152             : /// instructions (without using a separate calculation and a temp register).
     153             : /// Note that the exception here are vector stores/loads which cannot encode any
     154             : /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
     155             : static const unsigned DefaultSafeSPDisplacement = 255;
     156             : 
     157             : /// Look at each instruction that references stack frames and return the stack
     158             : /// size limit beyond which some of these instructions will require a scratch
     159             : /// register during their expansion later.
     160       14691 : static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
     161             :   // FIXME: For now, just conservatively guestimate based on unscaled indexing
     162             :   // range. We'll end up allocating an unnecessary spill slot a lot, but
     163             :   // realistically that's not a big deal at this stage of the game.
     164       32286 :   for (MachineBasicBlock &MBB : MF) {
     165       91935 :     for (MachineInstr &MI : MBB) {
     166       74245 :       if (MI.isDebugInstr() || MI.isPseudo() ||
     167       66991 :           MI.getOpcode() == AArch64::ADDXri ||
     168             :           MI.getOpcode() == AArch64::ADDSXri)
     169        8175 :         continue;
     170             : 
     171      246684 :       for (const MachineOperand &MO : MI.operands()) {
     172      180527 :         if (!MO.isFI())
     173      177215 :           continue;
     174             : 
     175        3312 :         int Offset = 0;
     176        3312 :         if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
     177             :             AArch64FrameOffsetCannotUpdate)
     178           8 :           return 0;
     179             :       }
     180             :     }
     181             :   }
     182             :   return DefaultSafeSPDisplacement;
     183             : }
     184             : 
     185       18480 : bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
     186       18480 :   if (!EnableRedZone)
     187             :     return false;
     188             :   // Don't use the red zone if the function explicitly asks us not to.
     189             :   // This is typically used for kernel code.
     190          94 :   if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
     191             :     return false;
     192             : 
     193          94 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     194             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     195          94 :   unsigned NumBytes = AFI->getLocalStackSize();
     196             : 
     197          94 :   return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
     198             : }
     199             : 
     200             : /// hasFP - Return true if the specified function should have a dedicated frame
     201             : /// pointer register.
     202      286816 : bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
     203      286816 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     204      286816 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     205             :   // Retain behavior of always omitting the FP for leaf functions when possible.
     206      286816 :   if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
     207             :     return true;
     208      281517 :   if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
     209      844146 :       MFI.hasStackMap() || MFI.hasPatchPoint() ||
     210      279998 :       RegInfo->needsStackRealignment(MF))
     211        3202 :     return true;
     212             :   // With large callframes around we may need to use FP to access the scavenging
     213             :   // emergency spillslot.
     214             :   //
     215             :   // Unfortunately some calls to hasFP() like machine verifier ->
     216             :   // getReservedReg() -> hasFP in the middle of global isel are too early
     217             :   // to know the max call frame size. Hopefully conservatively returning "true"
     218             :   // in those cases is fine.
     219             :   // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
     220      279545 :   if (!MFI.isMaxCallFrameSizeComputed() ||
     221             :       MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
     222       17684 :     return true;
     223             : 
     224             :   return false;
     225             : }
     226             : 
     227             : /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
     228             : /// not required, we reserve argument space for call sites in the function
     229             : /// immediately on entry to the current function.  This eliminates the need for
     230             : /// add/sub sp brackets around call sites.  Returns true if the call frame is
     231             : /// included as part of the stack frame.
     232             : bool
     233       10162 : AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
     234       10162 :   return !MF.getFrameInfo().hasVarSizedObjects();
     235             : }
     236             : 
     237        3860 : MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
     238             :     MachineFunction &MF, MachineBasicBlock &MBB,
     239             :     MachineBasicBlock::iterator I) const {
     240             :   const AArch64InstrInfo *TII =
     241        3860 :       static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
     242             :   DebugLoc DL = I->getDebugLoc();
     243        3860 :   unsigned Opc = I->getOpcode();
     244        3860 :   bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
     245        3860 :   uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
     246             : 
     247        3860 :   const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
     248        3860 :   if (!TFI->hasReservedCallFrame(MF)) {
     249         136 :     unsigned Align = getStackAlignment();
     250             : 
     251         136 :     int64_t Amount = I->getOperand(0).getImm();
     252         136 :     Amount = alignTo(Amount, Align);
     253         136 :     if (!IsDestroy)
     254          68 :       Amount = -Amount;
     255             : 
     256             :     // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
     257             :     // doesn't have to pop anything), then the first operand will be zero too so
     258             :     // this adjustment is a no-op.
     259         136 :     if (CalleePopAmount == 0) {
     260             :       // FIXME: in-function stack adjustment for calls is limited to 24-bits
     261             :       // because there's no guaranteed temporary register available.
     262             :       //
     263             :       // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
     264             :       // 1) For offset <= 12-bit, we use LSL #0
     265             :       // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
     266             :       // LSL #0, and the other uses LSL #12.
     267             :       //
     268             :       // Most call frames will be allocated at the start of a function so
     269             :       // this is OK, but it is a limitation that needs dealing with.
     270             :       assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
     271         135 :       emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
     272             :     }
     273        3724 :   } else if (CalleePopAmount != 0) {
     274             :     // If the calling convention demands that the callee pops arguments from the
     275             :     // stack, we want to add it back if we have a reserved call frame.
     276             :     assert(CalleePopAmount < 0xffffff && "call frame too large");
     277          12 :     emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
     278             :                     TII);
     279             :   }
     280        3860 :   return MBB.erase(I);
     281             : }
     282             : 
     283       29152 : static bool ShouldSignReturnAddress(MachineFunction &MF) {
     284             :   // The function should be signed in the following situations:
     285             :   // - sign-return-address=all
     286             :   // - sign-return-address=non-leaf and the functions spills the LR
     287             : 
     288       29152 :   const Function &F = MF.getFunction();
     289       29152 :   if (!F.hasFnAttribute("sign-return-address"))
     290             :     return false;
     291             : 
     292          16 :   StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
     293             :   if (Scope.equals("none"))
     294             :     return false;
     295             : 
     296             :   if (Scope.equals("all"))
     297             :     return true;
     298             : 
     299             :   assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
     300             : 
     301           6 :   for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
     302           4 :     if (Info.getReg() == AArch64::LR)
     303             :       return true;
     304             : 
     305             :   return false;
     306             : }
     307             : 
     308        1020 : void AArch64FrameLowering::emitCalleeSavedFrameMoves(
     309             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
     310        1020 :   MachineFunction &MF = *MBB.getParent();
     311        1020 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     312        1020 :   const TargetSubtargetInfo &STI = MF.getSubtarget();
     313        1020 :   const MCRegisterInfo *MRI = STI.getRegisterInfo();
     314        1020 :   const TargetInstrInfo *TII = STI.getInstrInfo();
     315             :   DebugLoc DL = MBB.findDebugLoc(MBBI);
     316             : 
     317             :   // Add callee saved registers to move list.
     318             :   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
     319        1020 :   if (CSI.empty())
     320             :     return;
     321             : 
     322        3431 :   for (const auto &Info : CSI) {
     323        2411 :     unsigned Reg = Info.getReg();
     324             :     int64_t Offset =
     325        2411 :         MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
     326        2411 :     unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
     327             :     unsigned CFIIndex = MF.addFrameInst(
     328        2411 :         MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     329        4822 :     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     330             :         .addCFIIndex(CFIIndex)
     331             :         .setMIFlags(MachineInstr::FrameSetup);
     332             :   }
     333             : }
     334             : 
     335             : // Find a scratch register that we can use at the start of the prologue to
     336             : // re-align the stack pointer.  We avoid using callee-save registers since they
     337             : // may appear to be free when this is called from canUseAsPrologue (during
     338             : // shrink wrapping), but then no longer be free when this is called from
     339             : // emitPrologue.
     340             : //
     341             : // FIXME: This is a bit conservative, since in the above case we could use one
     342             : // of the callee-save registers as a scratch temp to re-align the stack pointer,
     343             : // but we would then have to make sure that we were in fact saving at least one
     344             : // callee-save register in the prologue, which is additional complexity that
     345             : // doesn't seem worth the benefit.
     346          27 : static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
     347          27 :   MachineFunction *MF = MBB->getParent();
     348             : 
     349             :   // If MBB is an entry block, use X9 as the scratch register
     350          27 :   if (&MF->front() == MBB)
     351             :     return AArch64::X9;
     352             : 
     353           9 :   const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
     354             :   const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
     355           9 :   LivePhysRegs LiveRegs(TRI);
     356           9 :   LiveRegs.addLiveIns(*MBB);
     357             : 
     358             :   // Mark callee saved registers as used so we will not choose them.
     359           9 :   const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF);
     360         189 :   for (unsigned i = 0; CSRegs[i]; ++i)
     361         180 :     LiveRegs.addReg(CSRegs[i]);
     362             : 
     363             :   // Prefer X9 since it was historically used for the prologue scratch reg.
     364           9 :   const MachineRegisterInfo &MRI = MF->getRegInfo();
     365           9 :   if (LiveRegs.available(MRI, AArch64::X9))
     366             :     return AArch64::X9;
     367             : 
     368          37 :   for (unsigned Reg : AArch64::GPR64RegClass) {
     369          36 :     if (LiveRegs.available(MRI, Reg))
     370           2 :       return Reg;
     371             :   }
     372             :   return AArch64::NoRegister;
     373             : }
     374             : 
     375          78 : bool AArch64FrameLowering::canUseAsPrologue(
     376             :     const MachineBasicBlock &MBB) const {
     377          78 :   const MachineFunction *MF = MBB.getParent();
     378             :   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
     379          78 :   const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
     380             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     381             : 
     382             :   // Don't need a scratch register if we're not going to re-align the stack.
     383          78 :   if (!RegInfo->needsStackRealignment(*MF))
     384             :     return true;
     385             :   // Otherwise, we can use any block as long as it has a scratch register
     386             :   // available.
     387           7 :   return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
     388             : }
     389             : 
     390       30139 : static bool windowsRequiresStackProbe(MachineFunction &MF,
     391             :                                       unsigned StackSizeInBytes) {
     392       30139 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     393       30139 :   if (!Subtarget.isTargetWindows())
     394             :     return false;
     395         142 :   const Function &F = MF.getFunction();
     396             :   // TODO: When implementing stack protectors, take that into account
     397             :   // for the probe threshold.
     398         142 :   unsigned StackProbeSize = 4096;
     399         142 :   if (F.hasFnAttribute("stack-probe-size"))
     400           0 :     F.getFnAttribute("stack-probe-size")
     401           0 :         .getValueAsString()
     402           0 :         .getAsInteger(0, StackProbeSize);
     403         142 :   return (StackSizeInBytes >= StackProbeSize) &&
     404           6 :          !F.hasFnAttribute("no-stack-arg-probe");
     405             : }
     406             : 
     407       16020 : bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
     408             :     MachineFunction &MF, unsigned StackBumpBytes) const {
     409       16020 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     410       16020 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     411       16020 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     412             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     413             : 
     414       16020 :   if (AFI->getLocalStackSize() == 0)
     415             :     return false;
     416             : 
     417             :   // 512 is the maximum immediate for stp/ldp that will be used for
     418             :   // callee-save save/restores
     419        1343 :   if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
     420          50 :     return false;
     421             : 
     422        1293 :   if (MFI.hasVarSizedObjects())
     423             :     return false;
     424             : 
     425        1259 :   if (RegInfo->needsStackRealignment(MF))
     426             :     return false;
     427             : 
     428             :   // This isn't strictly necessary, but it simplifies things a bit since the
     429             :   // current RedZone handling code assumes the SP is adjusted by the
     430             :   // callee-save save/restore code.
     431        1235 :   if (canUseRedZone(MF))
     432           5 :     return false;
     433             : 
     434             :   return true;
     435             : }
     436             : 
     437             : // Convert callee-save register save/restore instruction to do stack pointer
     438             : // decrement/increment to allocate/deallocate the callee-save stack area by
     439             : // converting store/load to use pre/post increment version.
     440        1658 : static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     441             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     442             :     const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc) {
     443             :   // Ignore instructions that do not operate on SP, i.e. shadow call stack
     444             :   // instructions.
     445        3320 :   while (MBBI->getOpcode() == AArch64::STRXpost ||
     446             :          MBBI->getOpcode() == AArch64::LDRXpre) {
     447             :     assert(MBBI->getOperand(0).getReg() != AArch64::SP);
     448             :     ++MBBI;
     449             :   }
     450             : 
     451             :   unsigned NewOpc;
     452             :   int Scale = 1;
     453        1658 :   switch (MBBI->getOpcode()) {
     454           0 :   default:
     455           0 :     llvm_unreachable("Unexpected callee-save save/restore opcode!");
     456             :   case AArch64::STPXi:
     457             :     NewOpc = AArch64::STPXpre;
     458             :     Scale = 8;
     459             :     break;
     460          39 :   case AArch64::STPDi:
     461             :     NewOpc = AArch64::STPDpre;
     462             :     Scale = 8;
     463          39 :     break;
     464           4 :   case AArch64::STPQi:
     465             :     NewOpc = AArch64::STPQpre;
     466             :     Scale = 16;
     467           4 :     break;
     468         327 :   case AArch64::STRXui:
     469             :     NewOpc = AArch64::STRXpre;
     470         327 :     break;
     471          38 :   case AArch64::STRDui:
     472             :     NewOpc = AArch64::STRDpre;
     473          38 :     break;
     474           0 :   case AArch64::STRQui:
     475             :     NewOpc = AArch64::STRQpre;
     476           0 :     break;
     477         414 :   case AArch64::LDPXi:
     478             :     NewOpc = AArch64::LDPXpost;
     479             :     Scale = 8;
     480         414 :     break;
     481          39 :   case AArch64::LDPDi:
     482             :     NewOpc = AArch64::LDPDpost;
     483             :     Scale = 8;
     484          39 :     break;
     485           0 :   case AArch64::LDPQi:
     486             :     NewOpc = AArch64::LDPQpost;
     487             :     Scale = 16;
     488           0 :     break;
     489         337 :   case AArch64::LDRXui:
     490             :     NewOpc = AArch64::LDRXpost;
     491         337 :     break;
     492          38 :   case AArch64::LDRDui:
     493             :     NewOpc = AArch64::LDRDpost;
     494          38 :     break;
     495           0 :   case AArch64::LDRQui:
     496             :     NewOpc = AArch64::LDRQpost;
     497           0 :     break;
     498             :   }
     499             : 
     500        3316 :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
     501        1658 :   MIB.addReg(AArch64::SP, RegState::Define);
     502             : 
     503             :   // Copy all operands other than the immediate offset.
     504             :   unsigned OpndIdx = 0;
     505        5892 :   for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
     506             :        ++OpndIdx)
     507        4234 :     MIB.add(MBBI->getOperand(OpndIdx));
     508             : 
     509             :   assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
     510             :          "Unexpected immediate offset in first/last callee-save save/restore "
     511             :          "instruction!");
     512             :   assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
     513             :          "Unexpected base register in callee-save save/restore instruction!");
     514             :   assert(CSStackSizeInc % Scale == 0);
     515        1658 :   MIB.addImm(CSStackSizeInc / Scale);
     516             : 
     517        1658 :   MIB.setMIFlags(MBBI->getFlags());
     518        1658 :   MIB.setMemRefs(MBBI->memoperands());
     519             : 
     520        1658 :   return std::prev(MBB.erase(MBBI));
     521             : }
     522             : 
     523             : // Fixup callee-save register save/restore instructions to take into account
     524             : // combined SP bump by adding the local stack size to the stack offsets.
     525        1828 : static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
     526             :                                               unsigned LocalStackSize) {
     527        1828 :   unsigned Opc = MI.getOpcode();
     528             : 
     529             :   // Ignore instructions that do not operate on SP, i.e. shadow call stack
     530             :   // instructions.
     531        1828 :   if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre) {
     532             :     assert(MI.getOperand(0).getReg() != AArch64::SP);
     533             :     return;
     534             :   }
     535             : 
     536             :   unsigned Scale;
     537        1828 :   switch (Opc) {
     538             :   case AArch64::STPXi:
     539             :   case AArch64::STRXui:
     540             :   case AArch64::STPDi:
     541             :   case AArch64::STRDui:
     542             :   case AArch64::LDPXi:
     543             :   case AArch64::LDRXui:
     544             :   case AArch64::LDPDi:
     545             :   case AArch64::LDRDui:
     546             :     Scale = 8;
     547             :     break;
     548           8 :   case AArch64::STPQi:
     549             :   case AArch64::STRQui:
     550             :   case AArch64::LDPQi:
     551             :   case AArch64::LDRQui:
     552             :     Scale = 16;
     553           8 :     break;
     554           0 :   default:
     555           0 :     llvm_unreachable("Unexpected callee-save save/restore opcode!");
     556             :   }
     557             : 
     558        1828 :   unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
     559             :   assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
     560             :          "Unexpected base register in callee-save save/restore instruction!");
     561             :   // Last operand is immediate offset that needs fixing.
     562        1828 :   MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
     563             :   // All generated opcodes have scaled offsets.
     564             :   assert(LocalStackSize % Scale == 0);
     565        1828 :   OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
     566             : }
     567             : 
     568             : static void adaptForLdStOpt(MachineBasicBlock &MBB,
     569             :                             MachineBasicBlock::iterator FirstSPPopI,
     570             :                             MachineBasicBlock::iterator LastPopI) {
     571             :   // Sometimes (when we restore in the same order as we save), we can end up
     572             :   // with code like this:
     573             :   //
     574             :   // ldp      x26, x25, [sp]
     575             :   // ldp      x24, x23, [sp, #16]
     576             :   // ldp      x22, x21, [sp, #32]
     577             :   // ldp      x20, x19, [sp, #48]
     578             :   // add      sp, sp, #64
     579             :   //
     580             :   // In this case, it is always better to put the first ldp at the end, so
     581             :   // that the load-store optimizer can run and merge the ldp and the add into
     582             :   // a post-index ldp.
     583             :   // If we managed to grab the first pop instruction, move it to the end.
     584       13720 :   if (ReverseCSRRestoreSeq)
     585           4 :     MBB.splice(FirstSPPopI, &MBB, LastPopI);
     586             :   // We should end up with something like this now:
     587             :   //
     588             :   // ldp      x24, x23, [sp, #16]
     589             :   // ldp      x22, x21, [sp, #32]
     590             :   // ldp      x20, x19, [sp, #48]
     591             :   // ldp      x26, x25, [sp]
     592             :   // add      sp, sp, #64
     593             :   //
     594             :   // and the load-store optimizer can merge the last two instructions into:
     595             :   //
     596             :   // ldp      x26, x25, [sp], #64
     597             :   //
     598             : }
     599             : 
     600       14461 : void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
     601             :                                         MachineBasicBlock &MBB) const {
     602             :   MachineBasicBlock::iterator MBBI = MBB.begin();
     603       14461 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     604       14461 :   const Function &F = MF.getFunction();
     605       14461 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     606             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     607             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     608       14461 :   MachineModuleInfo &MMI = MF.getMMI();
     609       14461 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     610       14461 :   bool needsFrameMoves = MMI.hasDebugInfo() || F.needsUnwindTableEntry();
     611       14461 :   bool HasFP = hasFP(MF);
     612             : 
     613             :   // At this point, we're going to decide whether or not the function uses a
     614             :   // redzone. In most cases, the function doesn't have a redzone so let's
     615             :   // assume that's false and set it to true in the case that there's a redzone.
     616             :   AFI->setHasRedZone(false);
     617             : 
     618             :   // Debug location must be unknown since the first debug location is used
     619             :   // to determine the end of the prologue.
     620       14461 :   DebugLoc DL;
     621             : 
     622       14461 :   if (ShouldSignReturnAddress(MF)) {
     623          12 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
     624             :         .setMIFlag(MachineInstr::FrameSetup);
     625             :   }
     626             : 
     627             :   // All calls are tail calls in GHC calling conv, and functions have no
     628             :   // prologue/epilogue.
     629       28922 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     630             :     return;
     631             : 
     632       14455 :   int NumBytes = (int)MFI.getStackSize();
     633       14455 :   if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
     634             :     assert(!HasFP && "unexpected function without stack frame but with FP");
     635             : 
     636             :     // All of the stack allocation is for locals.
     637             :     AFI->setLocalStackSize(NumBytes);
     638             : 
     639       13126 :     if (!NumBytes)
     640             :       return;
     641             :     // REDZONE: If the stack size is less than 128 bytes, we don't need
     642             :     // to actually allocate.
     643         225 :     if (canUseRedZone(MF)) {
     644             :       AFI->setHasRedZone(true);
     645             :       ++NumRedZoneFunctions;
     646             :     } else {
     647         222 :       emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
     648             :                       MachineInstr::FrameSetup);
     649             : 
     650             :       // Label used to tie together the PROLOG_LABEL and the MachineMoves.
     651         222 :       MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
     652             :       // Encode the stack size of the leaf function.
     653             :       unsigned CFIIndex = MF.addFrameInst(
     654         222 :           MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
     655         444 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     656             :           .addCFIIndex(CFIIndex)
     657             :           .setMIFlags(MachineInstr::FrameSetup);
     658             :     }
     659         225 :     return;
     660             :   }
     661             : 
     662             :   bool IsWin64 =
     663        1329 :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
     664        1263 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
     665             : 
     666        1329 :   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
     667             :   // All of the remaining stack allocations are for locals.
     668        1329 :   AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
     669             : 
     670        1329 :   bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
     671        1329 :   if (CombineSPBump) {
     672         499 :     emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
     673             :                     MachineInstr::FrameSetup);
     674             :     NumBytes = 0;
     675         830 :   } else if (PrologueSaveSize != 0) {
     676             :     MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(MBB, MBBI, DL, TII,
     677         830 :                                                      -PrologueSaveSize);
     678         830 :     NumBytes -= PrologueSaveSize;
     679             :   }
     680             :   assert(NumBytes >= 0 && "Negative stack allocation size!?");
     681             : 
     682             :   // Move past the saves of the callee-saved registers, fixing up the offsets
     683             :   // and pre-inc if we decided to combine the callee-save and local stack
     684             :   // pointer bump above.
     685             :   MachineBasicBlock::iterator End = MBB.end();
     686        3519 :   while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
     687        2190 :     if (CombineSPBump)
     688         916 :       fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize());
     689             :     ++MBBI;
     690             :   }
     691        1329 :   if (HasFP) {
     692             :     // Only set up FP if we actually need to. Frame pointer is fp =
     693             :     // sp - fixedobject - 16.
     694         293 :     int FPOffset = AFI->getCalleeSavedStackSize() - 16;
     695         293 :     if (CombineSPBump)
     696         104 :       FPOffset += AFI->getLocalStackSize();
     697             : 
     698             :     // Issue    sub fp, sp, FPOffset or
     699             :     //          mov fp,sp          when FPOffset is zero.
     700             :     // Note: All stores of callee-saved registers are marked as "FrameSetup".
     701             :     // This code marks the instruction(s) that set the FP also.
     702         293 :     emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
     703             :                     MachineInstr::FrameSetup);
     704             :   }
     705             : 
     706        1329 :   if (windowsRequiresStackProbe(MF, NumBytes)) {
     707           2 :     uint32_t NumWords = NumBytes >> 4;
     708             : 
     709           4 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
     710           2 :         .addImm(NumWords)
     711             :         .setMIFlags(MachineInstr::FrameSetup);
     712             : 
     713           2 :     switch (MF.getTarget().getCodeModel()) {
     714           1 :     case CodeModel::Tiny:
     715             :     case CodeModel::Small:
     716             :     case CodeModel::Medium:
     717             :     case CodeModel::Kernel:
     718           2 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
     719             :           .addExternalSymbol("__chkstk")
     720           1 :           .addReg(AArch64::X15, RegState::Implicit)
     721             :           .setMIFlags(MachineInstr::FrameSetup);
     722           1 :       break;
     723           1 :     case CodeModel::Large:
     724           2 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
     725           1 :           .addReg(AArch64::X16, RegState::Define)
     726             :           .addExternalSymbol("__chkstk")
     727             :           .addExternalSymbol("__chkstk")
     728             :           .setMIFlags(MachineInstr::FrameSetup);
     729             : 
     730           2 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
     731           1 :           .addReg(AArch64::X16, RegState::Kill)
     732           1 :           .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
     733             :           .setMIFlags(MachineInstr::FrameSetup);
     734           1 :       break;
     735             :     }
     736             : 
     737           6 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
     738           2 :         .addReg(AArch64::SP, RegState::Kill)
     739           2 :         .addReg(AArch64::X15, RegState::Kill)
     740             :         .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
     741             :         .setMIFlags(MachineInstr::FrameSetup);
     742             :     NumBytes = 0;
     743             :   }
     744             : 
     745             :   // Allocate space for the rest of the frame.
     746        1327 :   if (NumBytes) {
     747          54 :     const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
     748             :     unsigned scratchSPReg = AArch64::SP;
     749             : 
     750          54 :     if (NeedsRealignment) {
     751          20 :       scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
     752             :       assert(scratchSPReg != AArch64::NoRegister);
     753             :     }
     754             : 
     755             :     // If we're a leaf function, try using the red zone.
     756          54 :     if (!canUseRedZone(MF))
     757             :       // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
     758             :       // the correct value here, as NumBytes also includes padding bytes,
     759             :       // which shouldn't be counted here.
     760          53 :       emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
     761             :                       MachineInstr::FrameSetup);
     762             : 
     763          54 :     if (NeedsRealignment) {
     764          20 :       const unsigned Alignment = MFI.getMaxAlignment();
     765          20 :       const unsigned NrBitsToZero = countTrailingZeros(Alignment);
     766             :       assert(NrBitsToZero > 1);
     767             :       assert(scratchSPReg != AArch64::SP);
     768             : 
     769             :       // SUB X9, SP, NumBytes
     770             :       //   -- X9 is temporary register, so shouldn't contain any live data here,
     771             :       //   -- free to use. This is already produced by emitFrameOffset above.
     772             :       // AND SP, X9, 0b11111...0000
     773             :       // The logical immediates have a non-trivial encoding. The following
     774             :       // formula computes the encoded immediate with all ones but
     775             :       // NrBitsToZero zero bits as least significant bits.
     776          20 :       uint32_t andMaskEncoded = (1 << 12)                         // = N
     777          20 :                                 | ((64 - NrBitsToZero) << 6)      // immr
     778          20 :                                 | ((64 - NrBitsToZero - 1) << 0); // imms
     779             : 
     780          60 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
     781          20 :           .addReg(scratchSPReg, RegState::Kill)
     782          20 :           .addImm(andMaskEncoded);
     783             :       AFI->setStackRealigned(true);
     784             :     }
     785             :   }
     786             : 
     787             :   // If we need a base pointer, set it up here. It's whatever the value of the
     788             :   // stack pointer is at this point. Any variable size objects will be allocated
     789             :   // after this, so we can still use the base pointer to reference locals.
     790             :   //
     791             :   // FIXME: Clarify FrameSetup flags here.
     792             :   // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
     793             :   // needed.
     794        1329 :   if (RegInfo->hasBasePointer(MF)) {
     795          15 :     TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
     796          15 :                      false);
     797             :   }
     798             : 
     799        1329 :   if (needsFrameMoves) {
     800        1020 :     const DataLayout &TD = MF.getDataLayout();
     801        1020 :     const int StackGrowth = -TD.getPointerSize(0);
     802        1020 :     unsigned FramePtr = RegInfo->getFrameRegister(MF);
     803             :     // An example of the prologue:
     804             :     //
     805             :     //     .globl __foo
     806             :     //     .align 2
     807             :     //  __foo:
     808             :     // Ltmp0:
     809             :     //     .cfi_startproc
     810             :     //     .cfi_personality 155, ___gxx_personality_v0
     811             :     // Leh_func_begin:
     812             :     //     .cfi_lsda 16, Lexception33
     813             :     //
     814             :     //     stp  xa,bx, [sp, -#offset]!
     815             :     //     ...
     816             :     //     stp  x28, x27, [sp, #offset-32]
     817             :     //     stp  fp, lr, [sp, #offset-16]
     818             :     //     add  fp, sp, #offset - 16
     819             :     //     sub  sp, sp, #1360
     820             :     //
     821             :     // The Stack:
     822             :     //       +-------------------------------------------+
     823             :     // 10000 | ........ | ........ | ........ | ........ |
     824             :     // 10004 | ........ | ........ | ........ | ........ |
     825             :     //       +-------------------------------------------+
     826             :     // 10008 | ........ | ........ | ........ | ........ |
     827             :     // 1000c | ........ | ........ | ........ | ........ |
     828             :     //       +===========================================+
     829             :     // 10010 |                X28 Register               |
     830             :     // 10014 |                X28 Register               |
     831             :     //       +-------------------------------------------+
     832             :     // 10018 |                X27 Register               |
     833             :     // 1001c |                X27 Register               |
     834             :     //       +===========================================+
     835             :     // 10020 |                Frame Pointer              |
     836             :     // 10024 |                Frame Pointer              |
     837             :     //       +-------------------------------------------+
     838             :     // 10028 |                Link Register              |
     839             :     // 1002c |                Link Register              |
     840             :     //       +===========================================+
     841             :     // 10030 | ........ | ........ | ........ | ........ |
     842             :     // 10034 | ........ | ........ | ........ | ........ |
     843             :     //       +-------------------------------------------+
     844             :     // 10038 | ........ | ........ | ........ | ........ |
     845             :     // 1003c | ........ | ........ | ........ | ........ |
     846             :     //       +-------------------------------------------+
     847             :     //
     848             :     //     [sp] = 10030        ::    >>initial value<<
     849             :     //     sp = 10020          ::  stp fp, lr, [sp, #-16]!
     850             :     //     fp = sp == 10020    ::  mov fp, sp
     851             :     //     [sp] == 10020       ::  stp x28, x27, [sp, #-16]!
     852             :     //     sp == 10010         ::    >>final value<<
     853             :     //
     854             :     // The frame pointer (w29) points to address 10020. If we use an offset of
     855             :     // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
     856             :     // for w27, and -32 for w28:
     857             :     //
     858             :     //  Ltmp1:
     859             :     //     .cfi_def_cfa w29, 16
     860             :     //  Ltmp2:
     861             :     //     .cfi_offset w30, -8
     862             :     //  Ltmp3:
     863             :     //     .cfi_offset w29, -16
     864             :     //  Ltmp4:
     865             :     //     .cfi_offset w27, -24
     866             :     //  Ltmp5:
     867             :     //     .cfi_offset w28, -32
     868             : 
     869        1020 :     if (HasFP) {
     870             :       // Define the current CFA rule to use the provided FP.
     871         186 :       unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
     872         372 :       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
     873         186 :           nullptr, Reg, 2 * StackGrowth - FixedObject));
     874         372 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     875             :           .addCFIIndex(CFIIndex)
     876             :           .setMIFlags(MachineInstr::FrameSetup);
     877             :     } else {
     878             :       // Encode the stack size of the leaf function.
     879             :       unsigned CFIIndex = MF.addFrameInst(
     880         834 :           MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize()));
     881        1668 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     882             :           .addCFIIndex(CFIIndex)
     883             :           .setMIFlags(MachineInstr::FrameSetup);
     884             :     }
     885             : 
     886             :     // Now emit the moves for whatever callee saved regs we have (including FP,
     887             :     // LR if those are saved).
     888        1020 :     emitCalleeSavedFrameMoves(MBB, MBBI);
     889             :   }
     890             : }
     891             : 
     892       14691 : static void InsertReturnAddressAuth(MachineFunction &MF,
     893             :                                     MachineBasicBlock &MBB) {
     894       14691 :   if (!ShouldSignReturnAddress(MF))
     895       14685 :     return;
     896           6 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     897             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     898             : 
     899           6 :   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
     900           6 :   DebugLoc DL;
     901           6 :   if (MBBI != MBB.end())
     902             :     DL = MBBI->getDebugLoc();
     903             : 
     904             :   // The AUTIASP instruction assembles to a hint instruction before v8.3a so
     905             :   // this instruction can safely used for any v8a architecture.
     906             :   // From v8.3a onwards there are optimised authenticate LR and return
     907             :   // instructions, namely RETA{A,B}, that can be used instead.
     908           6 :   if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
     909           1 :       MBBI->getOpcode() == AArch64::RET_ReallyLR) {
     910           2 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::RETAA)).copyImplicitOps(*MBBI);
     911           1 :     MBB.erase(MBBI);
     912             :   } else {
     913          10 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::AUTIASP))
     914             :         .setMIFlag(MachineInstr::FrameDestroy);
     915             :   }
     916             : }
     917             : 
     918       14697 : void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
     919             :                                         MachineBasicBlock &MBB) const {
     920       14697 :   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
     921       14697 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     922       14697 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     923             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     924       14697 :   DebugLoc DL;
     925             :   bool IsTailCallReturn = false;
     926       14697 :   if (MBB.end() != MBBI) {
     927             :     DL = MBBI->getDebugLoc();
     928       14697 :     unsigned RetOpcode = MBBI->getOpcode();
     929       14697 :     IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
     930             :       RetOpcode == AArch64::TCRETURNri;
     931             :   }
     932       14697 :   int NumBytes = MFI.getStackSize();
     933       14697 :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     934             : 
     935             :   // All calls are tail calls in GHC calling conv, and functions have no
     936             :   // prologue/epilogue.
     937       29394 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     938             :     return;
     939             : 
     940             :   // Initial and residual are named for consistency with the prologue. Note that
     941             :   // in the epilogue, the residual adjustment is executed first.
     942             :   uint64_t ArgumentPopSize = 0;
     943       14691 :   if (IsTailCallReturn) {
     944         202 :     MachineOperand &StackAdjust = MBBI->getOperand(1);
     945             : 
     946             :     // For a tail-call in a callee-pops-arguments environment, some or all of
     947             :     // the stack may actually be in use for the call's arguments, this is
     948             :     // calculated during LowerCall and consumed here...
     949         202 :     ArgumentPopSize = StackAdjust.getImm();
     950             :   } else {
     951             :     // ... otherwise the amount to pop is *all* of the argument space,
     952             :     // conveniently stored in the MachineFunctionInfo by
     953             :     // LowerFormalArguments. This will, of course, be zero for the C calling
     954             :     // convention.
     955       14489 :     ArgumentPopSize = AFI->getArgumentStackToRestore();
     956             :   }
     957             : 
     958             :   // The stack frame should be like below,
     959             :   //
     960             :   //      ----------------------                     ---
     961             :   //      |                    |                      |
     962             :   //      | BytesInStackArgArea|              CalleeArgStackSize
     963             :   //      | (NumReusableBytes) |                (of tail call)
     964             :   //      |                    |                     ---
     965             :   //      |                    |                      |
     966             :   //      ---------------------|        ---           |
     967             :   //      |                    |         |            |
     968             :   //      |   CalleeSavedReg   |         |            |
     969             :   //      | (CalleeSavedStackSize)|      |            |
     970             :   //      |                    |         |            |
     971             :   //      ---------------------|         |         NumBytes
     972             :   //      |                    |     StackSize  (StackAdjustUp)
     973             :   //      |   LocalStackSize   |         |            |
     974             :   //      | (covering callee   |         |            |
     975             :   //      |       args)        |         |            |
     976             :   //      |                    |         |            |
     977             :   //      ----------------------        ---          ---
     978             :   //
     979             :   // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
     980             :   //             = StackSize + ArgumentPopSize
     981             :   //
     982             :   // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
     983             :   // it as the 2nd argument of AArch64ISD::TC_RETURN.
     984             : 
     985       14691 :   auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
     986             : 
     987             :   bool IsWin64 =
     988             :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
     989       14603 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
     990             : 
     991             :   uint64_t AfterCSRPopSize = ArgumentPopSize;
     992       14691 :   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
     993       14691 :   bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
     994             :   // Assume we can't combine the last pop with the sp restore.
     995             : 
     996       14691 :   if (!CombineSPBump && PrologueSaveSize != 0) {
     997         834 :     MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
     998             :     // Converting the last ldp to a post-index ldp is valid only if the last
     999             :     // ldp's offset is 0.
    1000         834 :     const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
    1001             :     // If the offset is 0, convert it to a post-index ldp.
    1002         834 :     if (OffsetOp.getImm() == 0) {
    1003             :       convertCalleeSaveRestoreToSPPrePostIncDec(MBB, Pop, DL, TII,
    1004         828 :                                                 PrologueSaveSize);
    1005             :     } else {
    1006             :       // If not, make sure to emit an add after the last ldp.
    1007             :       // We're doing this by transfering the size to be restored from the
    1008             :       // adjustment *before* the CSR pops to the adjustment *after* the CSR
    1009             :       // pops.
    1010           6 :       AfterCSRPopSize += PrologueSaveSize;
    1011             :     }
    1012             :   }
    1013             : 
    1014             :   // Move past the restores of the callee-saved registers.
    1015             :   // If we plan on combining the sp bump of the local stack size and the callee
    1016             :   // save stack size, we might need to adjust the CSR save and restore offsets.
    1017       14691 :   MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
    1018             :   MachineBasicBlock::iterator Begin = MBB.begin();
    1019       16861 :   while (LastPopI != Begin) {
    1020             :     --LastPopI;
    1021       16336 :     if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
    1022             :       ++LastPopI;
    1023             :       break;
    1024        2170 :     } else if (CombineSPBump)
    1025         912 :       fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize());
    1026             :   }
    1027             : 
    1028             :   // If there is a single SP update, insert it before the ret and we're done.
    1029       14691 :   if (CombineSPBump) {
    1030         731 :     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
    1031         731 :                     NumBytes + AfterCSRPopSize, TII,
    1032             :                     MachineInstr::FrameDestroy);
    1033         731 :     return;
    1034             :   }
    1035             : 
    1036       13960 :   NumBytes -= PrologueSaveSize;
    1037             :   assert(NumBytes >= 0 && "Negative stack allocation size!?");
    1038             : 
    1039       13960 :   if (!hasFP(MF)) {
    1040       13773 :     bool RedZone = canUseRedZone(MF);
    1041             :     // If this was a redzone leaf function, we don't need to restore the
    1042             :     // stack pointer (but we may need to pop stack args for fastcc).
    1043       13773 :     if (RedZone && AfterCSRPopSize == 0)
    1044             :       return;
    1045             : 
    1046       13720 :     bool NoCalleeSaveRestore = PrologueSaveSize == 0;
    1047       13720 :     int StackRestoreBytes = RedZone ? 0 : NumBytes;
    1048       13720 :     if (NoCalleeSaveRestore)
    1049       13073 :       StackRestoreBytes += AfterCSRPopSize;
    1050             : 
    1051             :     // If we were able to combine the local stack pop with the argument pop,
    1052             :     // then we're done.
    1053       13720 :     bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
    1054             : 
    1055             :     // If we're done after this, make sure to help the load store optimizer.
    1056       13720 :     if (Done)
    1057       27422 :       adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
    1058             : 
    1059       13720 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
    1060             :                     StackRestoreBytes, TII, MachineInstr::FrameDestroy);
    1061       13720 :     if (Done)
    1062             :       return;
    1063             : 
    1064             :     NumBytes = 0;
    1065             :   }
    1066             : 
    1067             :   // Restore the original stack pointer.
    1068             :   // FIXME: Rather than doing the math here, we should instead just use
    1069             :   // non-post-indexed loads for the restores if we aren't actually going to
    1070             :   // be able to save any instructions.
    1071         196 :   if (MFI.hasVarSizedObjects() || AFI->isStackRealigned())
    1072          54 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
    1073          54 :                     -AFI->getCalleeSavedStackSize() + 16, TII,
    1074             :                     MachineInstr::FrameDestroy);
    1075         142 :   else if (NumBytes)
    1076           7 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
    1077             :                     MachineInstr::FrameDestroy);
    1078             : 
    1079             :   // This must be placed after the callee-save restore code because that code
    1080             :   // assumes the SP is at the same location as it was after the callee-save save
    1081             :   // code in the prologue.
    1082         196 :   if (AfterCSRPopSize) {
    1083             :     // Find an insertion point for the first ldp so that it goes before the
    1084             :     // shadow call stack epilog instruction. This ensures that the restore of
    1085             :     // lr from x18 is placed after the restore from sp.
    1086           9 :     auto FirstSPPopI = MBB.getFirstTerminator();
    1087          11 :     while (FirstSPPopI != Begin) {
    1088          11 :       auto Prev = std::prev(FirstSPPopI);
    1089          22 :       if (Prev->getOpcode() != AArch64::LDRXpre ||
    1090           2 :           Prev->getOperand(0).getReg() == AArch64::SP)
    1091             :         break;
    1092             :       FirstSPPopI = Prev;
    1093             :     }
    1094             : 
    1095           9 :     adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
    1096             : 
    1097           9 :     emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
    1098             :                     AfterCSRPopSize, TII, MachineInstr::FrameDestroy);
    1099             :   }
    1100             : }
    1101             : 
    1102             : /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
    1103             : /// debug info.  It's the same as what we use for resolving the code-gen
    1104             : /// references for now.  FIXME: This can go wrong when references are
    1105             : /// SP-relative and simple call frames aren't used.
    1106          14 : int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
    1107             :                                                  int FI,
    1108             :                                                  unsigned &FrameReg) const {
    1109          14 :   return resolveFrameIndexReference(MF, FI, FrameReg);
    1110             : }
    1111             : 
    1112        3561 : int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
    1113             :                                                      int FI, unsigned &FrameReg,
    1114             :                                                      bool PreferFP) const {
    1115        3561 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
    1116             :   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
    1117        3561 :       MF.getSubtarget().getRegisterInfo());
    1118             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1119        3561 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
    1120             :   bool IsWin64 =
    1121        3561 :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
    1122        3169 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
    1123        3561 :   int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16;
    1124        3561 :   int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
    1125             :   bool isFixed = MFI.isFixedObjectIndex(FI);
    1126        3189 :   bool isCSR = !isFixed && MFI.getObjectOffset(FI) >=
    1127        3189 :                                -((int)AFI->getCalleeSavedStackSize());
    1128             : 
    1129             :   // Use frame pointer to reference fixed objects. Use it for locals if
    1130             :   // there are VLAs or a dynamically realigned SP (and thus the SP isn't
    1131             :   // reliable as a base). Make sure useFPForScavengingIndex() does the
    1132             :   // right thing for the emergency spill slot.
    1133             :   bool UseFP = false;
    1134        3561 :   if (AFI->hasStackFrame()) {
    1135             :     // Note: Keeping the following as multiple 'if' statements rather than
    1136             :     // merging to a single expression for readability.
    1137             :     //
    1138             :     // Argument access should always use the FP.
    1139        2625 :     if (isFixed) {
    1140         194 :       UseFP = hasFP(MF);
    1141        2431 :     } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
    1142             :       // References to the CSR area must use FP if we're re-aligning the stack
    1143             :       // since the dynamically-sized alignment padding is between the SP/BP and
    1144             :       // the CSR area.
    1145             :       assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
    1146             :       UseFP = true;
    1147        2430 :     } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
    1148             :       // If the FPOffset is negative, we have to keep in mind that the
    1149             :       // available offset range for negative offsets is smaller than for
    1150             :       // positive ones. If an offset is
    1151             :       // available via the FP and the SP, use whichever is closest.
    1152         482 :       bool FPOffsetFits = FPOffset >= -256;
    1153         482 :       PreferFP |= Offset > -FPOffset;
    1154             : 
    1155         482 :       if (MFI.hasVarSizedObjects()) {
    1156             :         // If we have variable sized objects, we can use either FP or BP, as the
    1157             :         // SP offset is unknown. We can use the base pointer if we have one and
    1158             :         // FP is not preferred. If not, we're stuck with using FP.
    1159          46 :         bool CanUseBP = RegInfo->hasBasePointer(MF);
    1160          46 :         if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
    1161             :           UseFP = PreferFP;
    1162          40 :         else if (!CanUseBP) // Can't use BP. Forced to use FP.
    1163             :           UseFP = true;
    1164             :         // else we can use BP and FP, but the offset from FP won't fit.
    1165             :         // That will make us scavenge registers which we can probably avoid by
    1166             :         // using BP. If it won't fit for BP either, we'll scavenge anyway.
    1167         436 :       } else if (FPOffset >= 0) {
    1168             :         // Use SP or FP, whichever gives us the best chance of the offset
    1169             :         // being in range for direct access. If the FPOffset is positive,
    1170             :         // that'll always be best, as the SP will be even further away.
    1171             :         UseFP = true;
    1172             :       } else {
    1173             :         // We have the choice between FP and (SP or BP).
    1174         436 :         if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
    1175             :           UseFP = true;
    1176             :       }
    1177             :     }
    1178             :   }
    1179             : 
    1180             :   assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
    1181             :          "In the presence of dynamic stack pointer realignment, "
    1182             :          "non-argument/CSR objects cannot be accessed through the frame pointer");
    1183             : 
    1184         200 :   if (UseFP) {
    1185         357 :     FrameReg = RegInfo->getFrameRegister(MF);
    1186         357 :     return FPOffset;
    1187             :   }
    1188             : 
    1189             :   // Use the base pointer if we have one.
    1190        3204 :   if (RegInfo->hasBasePointer(MF))
    1191          11 :     FrameReg = RegInfo->getBaseRegister();
    1192             :   else {
    1193             :     assert(!MFI.hasVarSizedObjects() &&
    1194             :            "Can't use SP when we have var sized objects.");
    1195        3193 :     FrameReg = AArch64::SP;
    1196             :     // If we're using the red zone for this function, the SP won't actually
    1197             :     // be adjusted, so the offsets will be negative. They're also all
    1198             :     // within range of the signed 9-bit immediate instructions.
    1199        3193 :     if (canUseRedZone(MF))
    1200          23 :       Offset -= AFI->getLocalStackSize();
    1201             :   }
    1202             : 
    1203             :   return Offset;
    1204             : }
    1205             : 
    1206             : static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
    1207             :   // Do not set a kill flag on values that are also marked as live-in. This
    1208             :   // happens with the @llvm-returnaddress intrinsic and with arguments passed in
    1209             :   // callee saved registers.
    1210             :   // Omitting the kill flags is conservatively correct even if the live-in
    1211             :   // is not used after all.
    1212        3636 :   bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
    1213             :   return getKillRegState(!IsLiveIn);
    1214             : }
    1215             : 
    1216        3568 : static bool produceCompactUnwindFrame(MachineFunction &MF) {
    1217        3568 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
    1218        3568 :   AttributeList Attrs = MF.getFunction().getAttributes();
    1219        5209 :   return Subtarget.isTargetMachO() &&
    1220        3282 :          !(Subtarget.getTargetLowering()->supportSwiftError() &&
    1221        1641 :            Attrs.hasAttrSomewhere(Attribute::SwiftError));
    1222             : }
    1223             : 
    1224             : namespace {
    1225             : 
    1226             : struct RegPairInfo {
    1227             :   unsigned Reg1 = AArch64::NoRegister;
    1228             :   unsigned Reg2 = AArch64::NoRegister;
    1229             :   int FrameIdx;
    1230             :   int Offset;
    1231             :   enum RegType { GPR, FPR64, FPR128 } Type;
    1232             : 
    1233           0 :   RegPairInfo() = default;
    1234             : 
    1235           0 :   bool isPaired() const { return Reg2 != AArch64::NoRegister; }
    1236             : };
    1237             : 
    1238             : } // end anonymous namespace
    1239             : 
    1240           0 : static void computeCalleeSaveRegisterPairs(
    1241             :     MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
    1242             :     const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
    1243             :     bool &NeedShadowCallStackProlog) {
    1244             : 
    1245           0 :   if (CSI.empty())
    1246           0 :     return;
    1247             : 
    1248           0 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1249           0 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1250             :   CallingConv::ID CC = MF.getFunction().getCallingConv();
    1251           0 :   unsigned Count = CSI.size();
    1252             :   (void)CC;
    1253             :   // MachO's compact unwind format relies on all registers being stored in
    1254             :   // pairs.
    1255             :   assert((!produceCompactUnwindFrame(MF) ||
    1256             :           CC == CallingConv::PreserveMost ||
    1257             :           (Count & 1) == 0) &&
    1258             :          "Odd number of callee-saved regs to spill!");
    1259           0 :   int Offset = AFI->getCalleeSavedStackSize();
    1260             : 
    1261           0 :   for (unsigned i = 0; i < Count; ++i) {
    1262             :     RegPairInfo RPI;
    1263           0 :     RPI.Reg1 = CSI[i].getReg();
    1264             : 
    1265           0 :     if (AArch64::GPR64RegClass.contains(RPI.Reg1))
    1266           0 :       RPI.Type = RegPairInfo::GPR;
    1267           0 :     else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
    1268           0 :       RPI.Type = RegPairInfo::FPR64;
    1269           0 :     else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
    1270           0 :       RPI.Type = RegPairInfo::FPR128;
    1271             :     else
    1272           0 :       llvm_unreachable("Unsupported register class.");
    1273             : 
    1274             :     // Add the next reg to the pair if it is in the same register class.
    1275           0 :     if (i + 1 < Count) {
    1276           0 :       unsigned NextReg = CSI[i + 1].getReg();
    1277           0 :       switch (RPI.Type) {
    1278           0 :       case RegPairInfo::GPR:
    1279           0 :         if (AArch64::GPR64RegClass.contains(NextReg))
    1280           0 :           RPI.Reg2 = NextReg;
    1281             :         break;
    1282           0 :       case RegPairInfo::FPR64:
    1283           0 :         if (AArch64::FPR64RegClass.contains(NextReg))
    1284           0 :           RPI.Reg2 = NextReg;
    1285             :         break;
    1286           0 :       case RegPairInfo::FPR128:
    1287           0 :         if (AArch64::FPR128RegClass.contains(NextReg))
    1288           0 :           RPI.Reg2 = NextReg;
    1289             :         break;
    1290             :       }
    1291             :     }
    1292             : 
    1293             :     // If either of the registers to be saved is the lr register, it means that
    1294             :     // we also need to save lr in the shadow call stack.
    1295           0 :     if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
    1296           0 :         MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
    1297           0 :       if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
    1298           0 :         report_fatal_error("Must reserve x18 to use shadow call stack");
    1299           0 :       NeedShadowCallStackProlog = true;
    1300             :     }
    1301             : 
    1302             :     // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
    1303             :     // list to come in sorted by frame index so that we can issue the store
    1304             :     // pair instructions directly. Assert if we see anything otherwise.
    1305             :     //
    1306             :     // The order of the registers in the list is controlled by
    1307             :     // getCalleeSavedRegs(), so they will always be in-order, as well.
    1308             :     assert((!RPI.isPaired() ||
    1309             :             (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
    1310             :            "Out of order callee saved regs!");
    1311             : 
    1312             :     // MachO's compact unwind format relies on all registers being stored in
    1313             :     // adjacent register pairs.
    1314             :     assert((!produceCompactUnwindFrame(MF) ||
    1315             :             CC == CallingConv::PreserveMost ||
    1316             :             (RPI.isPaired() &&
    1317             :              ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
    1318             :               RPI.Reg1 + 1 == RPI.Reg2))) &&
    1319             :            "Callee-save registers not saved as adjacent register pair!");
    1320             : 
    1321           0 :     RPI.FrameIdx = CSI[i].getFrameIdx();
    1322             : 
    1323           0 :     int Scale = RPI.Type == RegPairInfo::FPR128 ? 16 : 8;
    1324           0 :     Offset -= RPI.isPaired() ? 2 * Scale : Scale;
    1325             : 
    1326             :     // Round up size of non-pair to pair size if we need to pad the
    1327             :     // callee-save area to ensure 16-byte alignment.
    1328           0 :     if (AFI->hasCalleeSaveStackFreeSpace() &&
    1329           0 :         RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired()) {
    1330           0 :       Offset -= 8;
    1331             :       assert(Offset % 16 == 0);
    1332             :       assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
    1333             :       MFI.setObjectAlignment(RPI.FrameIdx, 16);
    1334             :     }
    1335             : 
    1336             :     assert(Offset % Scale == 0);
    1337           0 :     RPI.Offset = Offset / Scale;
    1338             :     assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
    1339             :            "Offset out of bounds for LDP/STP immediate");
    1340             : 
    1341           0 :     RegPairs.push_back(RPI);
    1342           0 :     if (RPI.isPaired())
    1343             :       ++i;
    1344             :   }
    1345             : }
    1346             : 
    1347        1329 : bool AArch64FrameLowering::spillCalleeSavedRegisters(
    1348             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
    1349             :     const std::vector<CalleeSavedInfo> &CSI,
    1350             :     const TargetRegisterInfo *TRI) const {
    1351        1329 :   MachineFunction &MF = *MBB.getParent();
    1352        1329 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1353        1329 :   DebugLoc DL;
    1354        1329 :   SmallVector<RegPairInfo, 8> RegPairs;
    1355             : 
    1356        1329 :   bool NeedShadowCallStackProlog = false;
    1357        1329 :   computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
    1358             :                                  NeedShadowCallStackProlog);
    1359        1329 :   const MachineRegisterInfo &MRI = MF.getRegInfo();
    1360             : 
    1361        1329 :   if (NeedShadowCallStackProlog) {
    1362             :     // Shadow call stack prolog: str x30, [x18], #8
    1363           6 :     BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
    1364           2 :         .addReg(AArch64::X18, RegState::Define)
    1365           2 :         .addReg(AArch64::LR)
    1366           2 :         .addReg(AArch64::X18)
    1367             :         .addImm(8)
    1368             :         .setMIFlag(MachineInstr::FrameSetup);
    1369             : 
    1370             :     // This instruction also makes x18 live-in to the entry block.
    1371             :     MBB.addLiveIn(AArch64::X18);
    1372             :   }
    1373             : 
    1374        3514 :   for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
    1375             :        ++RPII) {
    1376        2185 :     RegPairInfo RPI = *RPII;
    1377             :     unsigned Reg1 = RPI.Reg1;
    1378             :     unsigned Reg2 = RPI.Reg2;
    1379             :     unsigned StrOpc;
    1380             : 
    1381             :     // Issue sequence of spills for cs regs.  The first spill may be converted
    1382             :     // to a pre-decrement store later by emitPrologue if the callee-save stack
    1383             :     // area allocation can't be combined with the local stack area allocation.
    1384             :     // For example:
    1385             :     //    stp     x22, x21, [sp, #0]     // addImm(+0)
    1386             :     //    stp     x20, x19, [sp, #16]    // addImm(+2)
    1387             :     //    stp     fp, lr, [sp, #32]      // addImm(+4)
    1388             :     // Rationale: This sequence saves uop updates compared to a sequence of
    1389             :     // pre-increment spills like stp xi,xj,[sp,#-16]!
    1390             :     // Note: Similar rationale and sequence for restores in epilog.
    1391             :     unsigned Size, Align;
    1392        2185 :     switch (RPI.Type) {
    1393        1897 :     case RegPairInfo::GPR:
    1394        1897 :        StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
    1395             :        Size = 8;
    1396             :        Align = 8;
    1397             :        break;
    1398         269 :     case RegPairInfo::FPR64:
    1399         269 :        StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
    1400             :        Size = 8;
    1401             :        Align = 8;
    1402             :        break;
    1403          19 :     case RegPairInfo::FPR128:
    1404          19 :        StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
    1405             :        Size = 16;
    1406             :        Align = 16;
    1407             :        break;
    1408             :     }
    1409             :     LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
    1410             :                if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
    1411             :                dbgs() << ") -> fi#(" << RPI.FrameIdx;
    1412             :                if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
    1413             :                dbgs() << ")\n");
    1414             : 
    1415        4370 :     MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
    1416        2185 :     if (!MRI.isReserved(Reg1))
    1417        2170 :       MBB.addLiveIn(Reg1);
    1418        2185 :     if (RPI.isPaired()) {
    1419        1451 :       if (!MRI.isReserved(Reg2))
    1420         975 :         MBB.addLiveIn(Reg2);
    1421        1451 :       MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
    1422             :       MIB.addMemOperand(MF.getMachineMemOperand(
    1423             :           MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
    1424        2902 :           MachineMemOperand::MOStore, Size, Align));
    1425             :     }
    1426        2185 :     MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
    1427        2185 :         .addReg(AArch64::SP)
    1428        2185 :         .addImm(RPI.Offset) // [sp, #offset*scale],
    1429             :                             // where factor*scale is implicit
    1430             :         .setMIFlag(MachineInstr::FrameSetup);
    1431             :     MIB.addMemOperand(MF.getMachineMemOperand(
    1432             :         MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
    1433        4370 :         MachineMemOperand::MOStore, Size, Align));
    1434             :   }
    1435        1329 :   return true;
    1436             : }
    1437             : 
    1438        1338 : bool AArch64FrameLowering::restoreCalleeSavedRegisters(
    1439             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
    1440             :     std::vector<CalleeSavedInfo> &CSI,
    1441             :     const TargetRegisterInfo *TRI) const {
    1442        1338 :   MachineFunction &MF = *MBB.getParent();
    1443        1338 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1444        1338 :   DebugLoc DL;
    1445        1338 :   SmallVector<RegPairInfo, 8> RegPairs;
    1446             : 
    1447        1338 :   if (MI != MBB.end())
    1448             :     DL = MI->getDebugLoc();
    1449             : 
    1450        1338 :   bool NeedShadowCallStackProlog = false;
    1451        1338 :   computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
    1452             :                                  NeedShadowCallStackProlog);
    1453             : 
    1454             :   auto EmitMI = [&](const RegPairInfo &RPI) {
    1455             :     unsigned Reg1 = RPI.Reg1;
    1456             :     unsigned Reg2 = RPI.Reg2;
    1457             : 
    1458             :     // Issue sequence of restores for cs regs. The last restore may be converted
    1459             :     // to a post-increment load later by emitEpilogue if the callee-save stack
    1460             :     // area allocation can't be combined with the local stack area allocation.
    1461             :     // For example:
    1462             :     //    ldp     fp, lr, [sp, #32]       // addImm(+4)
    1463             :     //    ldp     x20, x19, [sp, #16]     // addImm(+2)
    1464             :     //    ldp     x22, x21, [sp, #0]      // addImm(+0)
    1465             :     // Note: see comment in spillCalleeSavedRegisters()
    1466             :     unsigned LdrOpc;
    1467             :     unsigned Size, Align;
    1468             :     switch (RPI.Type) {
    1469             :     case RegPairInfo::GPR:
    1470             :        LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
    1471             :        Size = 8;
    1472             :        Align = 8;
    1473             :        break;
    1474             :     case RegPairInfo::FPR64:
    1475             :        LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
    1476             :        Size = 8;
    1477             :        Align = 8;
    1478             :        break;
    1479             :     case RegPairInfo::FPR128:
    1480             :        LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
    1481             :        Size = 16;
    1482             :        Align = 16;
    1483             :        break;
    1484             :     }
    1485             :     LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
    1486             :                if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
    1487             :                dbgs() << ") -> fi#(" << RPI.FrameIdx;
    1488             :                if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
    1489             :                dbgs() << ")\n");
    1490             : 
    1491             :     MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
    1492             :     if (RPI.isPaired()) {
    1493             :       MIB.addReg(Reg2, getDefRegState(true));
    1494             :       MIB.addMemOperand(MF.getMachineMemOperand(
    1495             :           MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
    1496             :           MachineMemOperand::MOLoad, Size, Align));
    1497             :     }
    1498             :     MIB.addReg(Reg1, getDefRegState(true))
    1499             :         .addReg(AArch64::SP)
    1500             :         .addImm(RPI.Offset) // [sp, #offset*scale]
    1501             :                             // where factor*scale is implicit
    1502             :         .setMIFlag(MachineInstr::FrameDestroy);
    1503             :     MIB.addMemOperand(MF.getMachineMemOperand(
    1504             :         MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
    1505             :         MachineMemOperand::MOLoad, Size, Align));
    1506        1338 :   };
    1507             : 
    1508        1338 :   if (ReverseCSRRestoreSeq)
    1509          26 :     for (const RegPairInfo &RPI : reverse(RegPairs))
    1510          20 :       EmitMI(RPI);
    1511             :   else
    1512        3480 :     for (const RegPairInfo &RPI : RegPairs)
    1513        2148 :       EmitMI(RPI);
    1514             : 
    1515        1338 :   if (NeedShadowCallStackProlog) {
    1516             :     // Shadow call stack epilog: ldr x30, [x18, #-8]!
    1517           6 :     BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
    1518           2 :         .addReg(AArch64::X18, RegState::Define)
    1519           2 :         .addReg(AArch64::LR, RegState::Define)
    1520           2 :         .addReg(AArch64::X18)
    1521             :         .addImm(-8)
    1522             :         .setMIFlag(MachineInstr::FrameDestroy);
    1523             :   }
    1524             : 
    1525        1338 :   return true;
    1526             : }
    1527             : 
    1528       14697 : void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
    1529             :                                                 BitVector &SavedRegs,
    1530             :                                                 RegScavenger *RS) const {
    1531             :   // All calls are tail calls in GHC calling conv, and functions have no
    1532             :   // prologue/epilogue.
    1533       29394 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
    1534             :     return;
    1535             : 
    1536       14691 :   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
    1537             :   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
    1538       14691 :       MF.getSubtarget().getRegisterInfo());
    1539       14691 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1540             :   unsigned UnspilledCSGPR = AArch64::NoRegister;
    1541             :   unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
    1542             : 
    1543       14691 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1544       14691 :   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
    1545             : 
    1546       14691 :   unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
    1547       14691 :                                 ? RegInfo->getBaseRegister()
    1548             :                                 : (unsigned)AArch64::NoRegister;
    1549             : 
    1550             :   unsigned ExtraCSSpill = 0;
    1551             :   // Figure out which callee-saved registers to save/restore.
    1552      308707 :   for (unsigned i = 0; CSRegs[i]; ++i) {
    1553      294016 :     const unsigned Reg = CSRegs[i];
    1554             : 
    1555             :     // Add the base pointer register to SavedRegs if it is callee-save.
    1556      294016 :     if (Reg == BasePointerReg)
    1557             :       SavedRegs.set(Reg);
    1558             : 
    1559             :     bool RegUsed = SavedRegs.test(Reg);
    1560      294016 :     unsigned PairedReg = CSRegs[i ^ 1];
    1561      294016 :     if (!RegUsed) {
    1562      463825 :       if (AArch64::GPR64RegClass.contains(Reg) &&
    1563      173339 :           !RegInfo->isReservedReg(MF, Reg)) {
    1564             :         UnspilledCSGPR = Reg;
    1565             :         UnspilledCSGPRPaired = PairedReg;
    1566             :       }
    1567      290486 :       continue;
    1568             :     }
    1569             : 
    1570             :     // MachO's compact unwind format relies on all registers being stored in
    1571             :     // pairs.
    1572             :     // FIXME: the usual format is actually better if unwinding isn't needed.
    1573        3530 :     if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
    1574             :         !SavedRegs.test(PairedReg)) {
    1575             :       SavedRegs.set(PairedReg);
    1576         763 :       if (AArch64::GPR64RegClass.contains(PairedReg) &&
    1577         374 :           !RegInfo->isReservedReg(MF, PairedReg))
    1578             :         ExtraCSSpill = PairedReg;
    1579             :     }
    1580             :   }
    1581             : 
    1582             :   // Calculates the callee saved stack size.
    1583             :   unsigned CSStackSize = 0;
    1584       14691 :   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1585       14691 :   const MachineRegisterInfo &MRI = MF.getRegInfo();
    1586       18241 :   for (unsigned Reg : SavedRegs.set_bits())
    1587        3550 :     CSStackSize += TRI->getRegSizeInBits(Reg, MRI) / 8;
    1588             : 
    1589             :   // Save number of saved regs, so we can easily update CSStackSize later.
    1590             :   unsigned NumSavedRegs = SavedRegs.count();
    1591             : 
    1592             :   // The frame record needs to be created by saving the appropriate registers
    1593       14691 :   unsigned EstimatedStackSize = MFI.estimateStackSize(MF);
    1594       29082 :   if (hasFP(MF) ||
    1595       14391 :       windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
    1596             :     SavedRegs.set(AArch64::FP);
    1597             :     SavedRegs.set(AArch64::LR);
    1598             :   }
    1599             : 
    1600             :   LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
    1601             :              for (unsigned Reg
    1602             :                   : SavedRegs.set_bits()) dbgs()
    1603             :              << ' ' << printReg(Reg, RegInfo);
    1604             :              dbgs() << "\n";);
    1605             : 
    1606             :   // If any callee-saved registers are used, the frame cannot be eliminated.
    1607             :   bool CanEliminateFrame = SavedRegs.count() == 0;
    1608             : 
    1609             :   // The CSR spill slots have not been allocated yet, so estimateStackSize
    1610             :   // won't include them.
    1611       14691 :   unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
    1612       14691 :   bool BigStack = (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
    1613       14691 :   if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
    1614             :     AFI->setHasStackFrame(true);
    1615             : 
    1616             :   // Estimate if we might need to scavenge a register at some point in order
    1617             :   // to materialize a stack offset. If so, either spill one additional
    1618             :   // callee-saved register or reserve a special spill slot to facilitate
    1619             :   // register scavenging. If we already spilled an extra callee-saved register
    1620             :   // above to keep the number of spills even, we don't need to do anything else
    1621             :   // here.
    1622       14691 :   if (BigStack) {
    1623          52 :     if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
    1624             :       LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
    1625             :                         << " to get a scratch register.\n");
    1626             :       SavedRegs.set(UnspilledCSGPR);
    1627             :       // MachO's compact unwind format relies on all registers being stored in
    1628             :       // pairs, so if we need to spill one extra for BigStack, then we need to
    1629             :       // store the pair.
    1630          38 :       if (produceCompactUnwindFrame(MF))
    1631             :         SavedRegs.set(UnspilledCSGPRPaired);
    1632             :       ExtraCSSpill = UnspilledCSGPRPaired;
    1633             :     }
    1634             : 
    1635             :     // If we didn't find an extra callee-saved register to spill, create
    1636             :     // an emergency spill slot.
    1637          52 :     if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
    1638          14 :       const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1639             :       const TargetRegisterClass &RC = AArch64::GPR64RegClass;
    1640             :       unsigned Size = TRI->getSpillSize(RC);
    1641             :       unsigned Align = TRI->getSpillAlignment(RC);
    1642          14 :       int FI = MFI.CreateStackObject(Size, Align, false);
    1643             :       RS->addScavengingFrameIndex(FI);
    1644             :       LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
    1645             :                         << " as the emergency spill slot.\n");
    1646             :     }
    1647             :   }
    1648             : 
    1649             :   // Adding the size of additional 64bit GPR saves.
    1650       14691 :   CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
    1651       29382 :   unsigned AlignedCSStackSize = alignTo(CSStackSize, 16);
    1652             :   LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
    1653             :                << EstimatedStackSize + AlignedCSStackSize
    1654             :                << " bytes.\n");
    1655             : 
    1656             :   // Round up to register pair alignment to avoid additional SP adjustment
    1657             :   // instructions.
    1658             :   AFI->setCalleeSavedStackSize(AlignedCSStackSize);
    1659       14691 :   AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
    1660             : }
    1661             : 
    1662         479 : bool AArch64FrameLowering::enableStackSlotScavenging(
    1663             :     const MachineFunction &MF) const {
    1664             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1665         479 :   return AFI->hasCalleeSaveStackFreeSpace();
    1666             : }

Generated by: LCOV version 1.13