LCOV - code coverage report
Current view: top level - lib/Target/AArch64 - AArch64FrameLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 456 462 98.7 %
Date: 2018-06-17 00:07:59 Functions: 25 25 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the AArch64 implementation of TargetFrameLowering class.
      11             : //
      12             : // On AArch64, stack frames are structured as follows:
      13             : //
      14             : // The stack grows downward.
      15             : //
      16             : // All of the individual frame areas on the frame below are optional, i.e. it's
      17             : // possible to create a function so that the particular area isn't present
      18             : // in the frame.
      19             : //
      20             : // At function entry, the "frame" looks as follows:
      21             : //
      22             : // |                                   | Higher address
      23             : // |-----------------------------------|
      24             : // |                                   |
      25             : // | arguments passed on the stack     |
      26             : // |                                   |
      27             : // |-----------------------------------| <- sp
      28             : // |                                   | Lower address
      29             : //
      30             : //
      31             : // After the prologue has run, the frame has the following general structure.
      32             : // Note that this doesn't depict the case where a red-zone is used. Also,
      33             : // technically the last frame area (VLAs) doesn't get created until in the
      34             : // main function body, after the prologue is run. However, it's depicted here
      35             : // for completeness.
      36             : //
      37             : // |                                   | Higher address
      38             : // |-----------------------------------|
      39             : // |                                   |
      40             : // | arguments passed on the stack     |
      41             : // |                                   |
      42             : // |-----------------------------------|
      43             : // |                                   |
      44             : // | (Win64 only) varargs from reg     |
      45             : // |                                   |
      46             : // |-----------------------------------|
      47             : // |                                   |
      48             : // | prev_fp, prev_lr                  |
      49             : // | (a.k.a. "frame record")           |
      50             : // |-----------------------------------| <- fp(=x29)
      51             : // |                                   |
      52             : // | other callee-saved registers      |
      53             : // |                                   |
      54             : // |-----------------------------------|
      55             : // |.empty.space.to.make.part.below....|
      56             : // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
      57             : // |.the.standard.16-byte.alignment....|  compile time; if present)
      58             : // |-----------------------------------|
      59             : // |                                   |
      60             : // | local variables of fixed size     |
      61             : // | including spill slots             |
      62             : // |-----------------------------------| <- bp(not defined by ABI,
      63             : // |.variable-sized.local.variables....|       LLVM chooses X19)
      64             : // |.(VLAs)............................| (size of this area is unknown at
      65             : // |...................................|  compile time)
      66             : // |-----------------------------------| <- sp
      67             : // |                                   | Lower address
      68             : //
      69             : //
      70             : // To access the data in a frame, at-compile time, a constant offset must be
      71             : // computable from one of the pointers (fp, bp, sp) to access it. The size
      72             : // of the areas with a dotted background cannot be computed at compile-time
      73             : // if they are present, making it required to have all three of fp, bp and
      74             : // sp to be set up to be able to access all contents in the frame areas,
      75             : // assuming all of the frame areas are non-empty.
      76             : //
      77             : // For most functions, some of the frame areas are empty. For those functions,
      78             : // it may not be necessary to set up fp or bp:
      79             : // * A base pointer is definitely needed when there are both VLAs and local
      80             : //   variables with more-than-default alignment requirements.
      81             : // * A frame pointer is definitely needed when there are local variables with
      82             : //   more-than-default alignment requirements.
      83             : //
      84             : // In some cases when a base pointer is not strictly needed, it is generated
      85             : // anyway when offsets from the frame pointer to access local variables become
      86             : // so large that the offset can't be encoded in the immediate fields of loads
      87             : // or stores.
      88             : //
      89             : // FIXME: also explain the redzone concept.
      90             : // FIXME: also explain the concept of reserved call frames.
      91             : //
      92             : //===----------------------------------------------------------------------===//
      93             : 
      94             : #include "AArch64FrameLowering.h"
      95             : #include "AArch64InstrInfo.h"
      96             : #include "AArch64MachineFunctionInfo.h"
      97             : #include "AArch64RegisterInfo.h"
      98             : #include "AArch64Subtarget.h"
      99             : #include "AArch64TargetMachine.h"
     100             : #include "MCTargetDesc/AArch64AddressingModes.h"
     101             : #include "llvm/ADT/SmallVector.h"
     102             : #include "llvm/ADT/Statistic.h"
     103             : #include "llvm/CodeGen/LivePhysRegs.h"
     104             : #include "llvm/CodeGen/MachineBasicBlock.h"
     105             : #include "llvm/CodeGen/MachineFrameInfo.h"
     106             : #include "llvm/CodeGen/MachineFunction.h"
     107             : #include "llvm/CodeGen/MachineInstr.h"
     108             : #include "llvm/CodeGen/MachineInstrBuilder.h"
     109             : #include "llvm/CodeGen/MachineMemOperand.h"
     110             : #include "llvm/CodeGen/MachineModuleInfo.h"
     111             : #include "llvm/CodeGen/MachineOperand.h"
     112             : #include "llvm/CodeGen/MachineRegisterInfo.h"
     113             : #include "llvm/CodeGen/RegisterScavenging.h"
     114             : #include "llvm/CodeGen/TargetInstrInfo.h"
     115             : #include "llvm/CodeGen/TargetRegisterInfo.h"
     116             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
     117             : #include "llvm/IR/Attributes.h"
     118             : #include "llvm/IR/CallingConv.h"
     119             : #include "llvm/IR/DataLayout.h"
     120             : #include "llvm/IR/DebugLoc.h"
     121             : #include "llvm/IR/Function.h"
     122             : #include "llvm/MC/MCDwarf.h"
     123             : #include "llvm/Support/CommandLine.h"
     124             : #include "llvm/Support/Debug.h"
     125             : #include "llvm/Support/ErrorHandling.h"
     126             : #include "llvm/Support/MathExtras.h"
     127             : #include "llvm/Support/raw_ostream.h"
     128             : #include "llvm/Target/TargetMachine.h"
     129             : #include "llvm/Target/TargetOptions.h"
     130             : #include <cassert>
     131             : #include <cstdint>
     132             : #include <iterator>
     133             : #include <vector>
     134             : 
     135             : using namespace llvm;
     136             : 
     137             : #define DEBUG_TYPE "frame-info"
     138             : 
     139      101169 : static cl::opt<bool> EnableRedZone("aarch64-redzone",
     140      101169 :                                    cl::desc("enable use of redzone on AArch64"),
     141      303507 :                                    cl::init(false), cl::Hidden);
     142             : 
     143             : static cl::opt<bool>
     144      101169 :     ReverseCSRRestoreSeq("reverse-csr-restore-seq",
     145      101169 :                          cl::desc("reverse the CSR restore sequence"),
     146      303507 :                          cl::init(false), cl::Hidden);
     147             : 
     148             : STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
     149             : 
     150             : /// This is the biggest offset to the stack pointer we can encode in aarch64
     151             : /// instructions (without using a separate calculation and a temp register).
     152             : /// Note that the exception here are vector stores/loads which cannot encode any
     153             : /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
     154             : static const unsigned DefaultSafeSPDisplacement = 255;
     155             : 
     156             : /// Look at each instruction that references stack frames and return the stack
     157             : /// size limit beyond which some of these instructions will require a scratch
     158             : /// register during their expansion later.
     159       14174 : static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
     160             :   // FIXME: For now, just conservatively guestimate based on unscaled indexing
     161             :   // range. We'll end up allocating an unnecessary spill slot a lot, but
     162             :   // realistically that's not a big deal at this stage of the game.
     163       31194 :   for (MachineBasicBlock &MBB : MF) {
     164      104347 :     for (MachineInstr &MI : MBB) {
     165      141260 :       if (MI.isDebugInstr() || MI.isPseudo() ||
     166      125842 :           MI.getOpcode() == AArch64::ADDXri ||
     167             :           MI.getOpcode() == AArch64::ADDSXri)
     168        7747 :         continue;
     169             : 
     170      402492 :       for (const MachineOperand &MO : MI.operands()) {
     171      169978 :         if (!MO.isFI())
     172      166899 :           continue;
     173             : 
     174        3079 :         int Offset = 0;
     175        3079 :         if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
     176             :             AArch64FrameOffsetCannotUpdate)
     177           8 :           return 0;
     178             :       }
     179             :     }
     180             :   }
     181             :   return DefaultSafeSPDisplacement;
     182             : }
     183             : 
     184       17688 : bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
     185       17688 :   if (!EnableRedZone)
     186             :     return false;
     187             :   // Don't use the red zone if the function explicitly asks us not to.
     188             :   // This is typically used for kernel code.
     189         188 :   if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
     190             :     return false;
     191             : 
     192          94 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     193             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     194          94 :   unsigned NumBytes = AFI->getLocalStackSize();
     195             : 
     196          94 :   return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128);
     197             : }
     198             : 
     199             : /// hasFP - Return true if the specified function should have a dedicated frame
     200             : /// pointer register.
     201      104218 : bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
     202      104218 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     203      104218 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     204             :   // Retain behavior of always omitting the FP for leaf functions when possible.
     205      104218 :   if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
     206             :     return true;
     207      305560 :   if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
     208      406136 :       MFI.hasStackMap() || MFI.hasPatchPoint() ||
     209      100892 :       RegInfo->needsStackRealignment(MF))
     210             :     return true;
     211             :   // With large callframes around we may need to use FP to access the scavenging
     212             :   // emergency spillslot.
     213             :   //
     214             :   // Unfortunately some calls to hasFP() like machine verifier ->
     215             :   // getReservedReg() -> hasFP in the middle of global isel are too early
     216             :   // to know the max call frame size. Hopefully conservatively returning "true"
     217             :   // in those cases is fine.
     218             :   // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
     219      100696 :   if (!MFI.isMaxCallFrameSizeComputed() ||
     220             :       MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
     221             :     return true;
     222             : 
     223             :   return false;
     224             : }
     225             : 
     226             : /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
     227             : /// not required, we reserve argument space for call sites in the function
     228             : /// immediately on entry to the current function.  This eliminates the need for
     229             : /// add/sub sp brackets around call sites.  Returns true if the call frame is
     230             : /// included as part of the stack frame.
     231             : bool
     232       10840 : AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
     233       10840 :   return !MF.getFrameInfo().hasVarSizedObjects();
     234             : }
     235             : 
     236        3676 : MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
     237             :     MachineFunction &MF, MachineBasicBlock &MBB,
     238             :     MachineBasicBlock::iterator I) const {
     239             :   const AArch64InstrInfo *TII =
     240        3676 :       static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
     241             :   DebugLoc DL = I->getDebugLoc();
     242        3676 :   unsigned Opc = I->getOpcode();
     243        3676 :   bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
     244        3676 :   uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
     245             : 
     246        3676 :   const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
     247        3676 :   if (!TFI->hasReservedCallFrame(MF)) {
     248         136 :     unsigned Align = getStackAlignment();
     249             : 
     250         136 :     int64_t Amount = I->getOperand(0).getImm();
     251         272 :     Amount = alignTo(Amount, Align);
     252         136 :     if (!IsDestroy)
     253          68 :       Amount = -Amount;
     254             : 
     255             :     // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
     256             :     // doesn't have to pop anything), then the first operand will be zero too so
     257             :     // this adjustment is a no-op.
     258         136 :     if (CalleePopAmount == 0) {
     259             :       // FIXME: in-function stack adjustment for calls is limited to 24-bits
     260             :       // because there's no guaranteed temporary register available.
     261             :       //
     262             :       // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
     263             :       // 1) For offset <= 12-bit, we use LSL #0
     264             :       // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
     265             :       // LSL #0, and the other uses LSL #12.
     266             :       //
     267             :       // Most call frames will be allocated at the start of a function so
     268             :       // this is OK, but it is a limitation that needs dealing with.
     269             :       assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
     270         135 :       emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, Amount, TII);
     271             :     }
     272        3540 :   } else if (CalleePopAmount != 0) {
     273             :     // If the calling convention demands that the callee pops arguments from the
     274             :     // stack, we want to add it back if we have a reserved call frame.
     275             :     assert(CalleePopAmount < 0xffffff && "call frame too large");
     276          12 :     emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, -CalleePopAmount,
     277             :                     TII);
     278             :   }
     279        7352 :   return MBB.erase(I);
     280             : }
     281             : 
     282         963 : void AArch64FrameLowering::emitCalleeSavedFrameMoves(
     283             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
     284         963 :   MachineFunction &MF = *MBB.getParent();
     285         963 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     286         963 :   const TargetSubtargetInfo &STI = MF.getSubtarget();
     287         963 :   const MCRegisterInfo *MRI = STI.getRegisterInfo();
     288         963 :   const TargetInstrInfo *TII = STI.getInstrInfo();
     289             :   DebugLoc DL = MBB.findDebugLoc(MBBI);
     290             : 
     291             :   // Add callee saved registers to move list.
     292             :   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
     293         963 :   if (CSI.empty())
     294             :     return;
     295             : 
     296        3199 :   for (const auto &Info : CSI) {
     297        2236 :     unsigned Reg = Info.getReg();
     298             :     int64_t Offset =
     299        4472 :         MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
     300        2236 :     unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
     301             :     unsigned CFIIndex = MF.addFrameInst(
     302        4472 :         MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     303        6708 :     BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     304             :         .addCFIIndex(CFIIndex)
     305             :         .setMIFlags(MachineInstr::FrameSetup);
     306             :   }
     307             : }
     308             : 
     309             : // Find a scratch register that we can use at the start of the prologue to
     310             : // re-align the stack pointer.  We avoid using callee-save registers since they
     311             : // may appear to be free when this is called from canUseAsPrologue (during
     312             : // shrink wrapping), but then no longer be free when this is called from
     313             : // emitPrologue.
     314             : //
     315             : // FIXME: This is a bit conservative, since in the above case we could use one
     316             : // of the callee-save registers as a scratch temp to re-align the stack pointer,
     317             : // but we would then have to make sure that we were in fact saving at least one
     318             : // callee-save register in the prologue, which is additional complexity that
     319             : // doesn't seem worth the benefit.
     320          27 : static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
     321          27 :   MachineFunction *MF = MBB->getParent();
     322             : 
     323             :   // If MBB is an entry block, use X9 as the scratch register
     324          27 :   if (&MF->front() == MBB)
     325             :     return AArch64::X9;
     326             : 
     327           9 :   const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
     328             :   const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
     329           9 :   LivePhysRegs LiveRegs(TRI);
     330           9 :   LiveRegs.addLiveIns(*MBB);
     331             : 
     332             :   // Mark callee saved registers as used so we will not choose them.
     333           9 :   const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF);
     334         369 :   for (unsigned i = 0; CSRegs[i]; ++i)
     335         180 :     LiveRegs.addReg(CSRegs[i]);
     336             : 
     337             :   // Prefer X9 since it was historically used for the prologue scratch reg.
     338           9 :   const MachineRegisterInfo &MRI = MF->getRegInfo();
     339           9 :   if (LiveRegs.available(MRI, AArch64::X9))
     340             :     return AArch64::X9;
     341             : 
     342          74 :   for (unsigned Reg : AArch64::GPR64RegClass) {
     343          36 :     if (LiveRegs.available(MRI, Reg))
     344             :       return Reg;
     345             :   }
     346             :   return AArch64::NoRegister;
     347             : }
     348             : 
     349          78 : bool AArch64FrameLowering::canUseAsPrologue(
     350             :     const MachineBasicBlock &MBB) const {
     351          78 :   const MachineFunction *MF = MBB.getParent();
     352             :   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
     353          78 :   const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
     354             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     355             : 
     356             :   // Don't need a scratch register if we're not going to re-align the stack.
     357          78 :   if (!RegInfo->needsStackRealignment(*MF))
     358             :     return true;
     359             :   // Otherwise, we can use any block as long as it has a scratch register
     360             :   // available.
     361           7 :   return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
     362             : }
     363             : 
     364       29057 : static bool windowsRequiresStackProbe(MachineFunction &MF,
     365             :                                       unsigned StackSizeInBytes) {
     366       29057 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     367       29057 :   if (!Subtarget.isTargetWindows())
     368             :     return false;
     369         120 :   const Function &F = MF.getFunction();
     370             :   // TODO: When implementing stack protectors, take that into account
     371             :   // for the probe threshold.
     372         120 :   unsigned StackProbeSize = 4096;
     373         120 :   if (F.hasFnAttribute("stack-probe-size"))
     374           0 :     F.getFnAttribute("stack-probe-size")
     375           0 :         .getValueAsString()
     376           0 :         .getAsInteger(0, StackProbeSize);
     377         126 :   return (StackSizeInBytes >= StackProbeSize) &&
     378             :          !F.hasFnAttribute("no-stack-arg-probe");
     379             : }
     380             : 
     381       15460 : bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
     382             :     MachineFunction &MF, unsigned StackBumpBytes) const {
     383       15460 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     384       15460 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     385       15460 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     386             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     387             : 
     388       15460 :   if (AFI->getLocalStackSize() == 0)
     389             :     return false;
     390             : 
     391             :   // 512 is the maximum immediate for stp/ldp that will be used for
     392             :   // callee-save save/restores
     393        1249 :   if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
     394             :     return false;
     395             : 
     396        1200 :   if (MFI.hasVarSizedObjects())
     397             :     return false;
     398             : 
     399        1166 :   if (RegInfo->needsStackRealignment(MF))
     400             :     return false;
     401             : 
     402             :   // This isn't strictly necessary, but it simplifies things a bit since the
     403             :   // current RedZone handling code assumes the SP is adjusted by the
     404             :   // callee-save save/restore code.
     405        1142 :   if (canUseRedZone(MF))
     406             :     return false;
     407             : 
     408        1137 :   return true;
     409             : }
     410             : 
     411             : // Convert callee-save register save/restore instruction to do stack pointer
     412             : // decrement/increment to allocate/deallocate the callee-save stack area by
     413             : // converting store/load to use pre/post increment version.
     414        1594 : static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
     415             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     416             :     const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc) {
     417             :   // Ignore instructions that do not operate on SP, i.e. shadow call stack
     418             :   // instructions.
     419        3192 :   while (MBBI->getOpcode() == AArch64::STRXpost ||
     420             :          MBBI->getOpcode() == AArch64::LDRXpre) {
     421             :     assert(MBBI->getOperand(0).getReg() != AArch64::SP);
     422             :     ++MBBI;
     423             :   }
     424             : 
     425             :   unsigned NewOpc;
     426             :   bool NewIsUnscaled = false;
     427        1594 :   switch (MBBI->getOpcode()) {
     428           0 :   default:
     429           0 :     llvm_unreachable("Unexpected callee-save save/restore opcode!");
     430             :   case AArch64::STPXi:
     431             :     NewOpc = AArch64::STPXpre;
     432             :     break;
     433          36 :   case AArch64::STPDi:
     434             :     NewOpc = AArch64::STPDpre;
     435          36 :     break;
     436         306 :   case AArch64::STRXui:
     437             :     NewOpc = AArch64::STRXpre;
     438             :     NewIsUnscaled = true;
     439         306 :     break;
     440          36 :   case AArch64::STRDui:
     441             :     NewOpc = AArch64::STRDpre;
     442             :     NewIsUnscaled = true;
     443          36 :     break;
     444         410 :   case AArch64::LDPXi:
     445             :     NewOpc = AArch64::LDPXpost;
     446         410 :     break;
     447          36 :   case AArch64::LDPDi:
     448             :     NewOpc = AArch64::LDPDpost;
     449          36 :     break;
     450         316 :   case AArch64::LDRXui:
     451             :     NewOpc = AArch64::LDRXpost;
     452             :     NewIsUnscaled = true;
     453         316 :     break;
     454          36 :   case AArch64::LDRDui:
     455             :     NewOpc = AArch64::LDRDpost;
     456             :     NewIsUnscaled = true;
     457          36 :     break;
     458             :   }
     459             : 
     460        3188 :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
     461        1594 :   MIB.addReg(AArch64::SP, RegState::Define);
     462             : 
     463             :   // Copy all operands other than the immediate offset.
     464             :   unsigned OpndIdx = 0;
     465        5682 :   for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
     466             :        ++OpndIdx)
     467        4088 :     MIB.add(MBBI->getOperand(OpndIdx));
     468             : 
     469             :   assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
     470             :          "Unexpected immediate offset in first/last callee-save save/restore "
     471             :          "instruction!");
     472             :   assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
     473             :          "Unexpected base register in callee-save save/restore instruction!");
     474             :   // Last operand is immediate offset that needs fixing.
     475             :   assert(CSStackSizeInc % 8 == 0);
     476        1594 :   int64_t CSStackSizeIncImm = CSStackSizeInc;
     477        1594 :   if (!NewIsUnscaled)
     478         900 :     CSStackSizeIncImm /= 8;
     479             :   MIB.addImm(CSStackSizeIncImm);
     480             : 
     481        1594 :   MIB.setMIFlags(MBBI->getFlags());
     482        1594 :   MIB.setMemRefs(MBBI->memoperands_begin(), MBBI->memoperands_end());
     483             : 
     484        3188 :   return std::prev(MBB.erase(MBBI));
     485             : }
     486             : 
     487             : // Fixup callee-save register save/restore instructions to take into account
     488             : // combined SP bump by adding the local stack size to the stack offsets.
     489        1620 : static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
     490             :                                               unsigned LocalStackSize) {
     491        1620 :   unsigned Opc = MI.getOpcode();
     492             : 
     493             :   // Ignore instructions that do not operate on SP, i.e. shadow call stack
     494             :   // instructions.
     495        1620 :   if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre) {
     496             :     assert(MI.getOperand(0).getReg() != AArch64::SP);
     497             :     return;
     498             :   }
     499             : 
     500             :   (void)Opc;
     501             :   assert((Opc == AArch64::STPXi || Opc == AArch64::STPDi ||
     502             :           Opc == AArch64::STRXui || Opc == AArch64::STRDui ||
     503             :           Opc == AArch64::LDPXi || Opc == AArch64::LDPDi ||
     504             :           Opc == AArch64::LDRXui || Opc == AArch64::LDRDui) &&
     505             :          "Unexpected callee-save save/restore opcode!");
     506             : 
     507        1620 :   unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
     508             :   assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
     509             :          "Unexpected base register in callee-save save/restore instruction!");
     510             :   // Last operand is immediate offset that needs fixing.
     511        1620 :   MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
     512             :   // All generated opcodes have scaled offsets.
     513             :   assert(LocalStackSize % 8 == 0);
     514        1620 :   OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / 8);
     515             : }
     516             : 
     517             : static void adaptForLdStOpt(MachineBasicBlock &MBB,
     518             :                             MachineBasicBlock::iterator FirstSPPopI,
     519             :                             MachineBasicBlock::iterator LastPopI) {
     520             :   // Sometimes (when we restore in the same order as we save), we can end up
     521             :   // with code like this:
     522             :   //
     523             :   // ldp      x26, x25, [sp]
     524             :   // ldp      x24, x23, [sp, #16]
     525             :   // ldp      x22, x21, [sp, #32]
     526             :   // ldp      x20, x19, [sp, #48]
     527             :   // add      sp, sp, #64
     528             :   //
     529             :   // In this case, it is always better to put the first ldp at the end, so
     530             :   // that the load-store optimizer can run and merge the ldp and the add into
     531             :   // a post-index ldp.
     532             :   // If we managed to grab the first pop instruction, move it to the end.
     533       13287 :   if (ReverseCSRRestoreSeq)
     534           4 :     MBB.splice(FirstSPPopI, &MBB, LastPopI);
     535             :   // We should end up with something like this now:
     536             :   //
     537             :   // ldp      x24, x23, [sp, #16]
     538             :   // ldp      x22, x21, [sp, #32]
     539             :   // ldp      x20, x19, [sp, #48]
     540             :   // ldp      x26, x25, [sp]
     541             :   // add      sp, sp, #64
     542             :   //
     543             :   // and the load-store optimizer can merge the last two instructions into:
     544             :   //
     545             :   // ldp      x26, x25, [sp], #64
     546             :   //
     547             : }
     548             : 
     549       13980 : void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
     550             :                                         MachineBasicBlock &MBB) const {
     551       13980 :   MachineBasicBlock::iterator MBBI = MBB.begin();
     552       13980 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     553       13980 :   const Function &F = MF.getFunction();
     554       13980 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     555             :   const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
     556             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     557       13980 :   MachineModuleInfo &MMI = MF.getMMI();
     558       13980 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     559       13980 :   bool needsFrameMoves = MMI.hasDebugInfo() || F.needsUnwindTableEntry();
     560       13980 :   bool HasFP = hasFP(MF);
     561             : 
     562             :   // At this point, we're going to decide whether or not the function uses a
     563             :   // redzone. In most cases, the function doesn't have a redzone so let's
     564             :   // assume that's false and set it to true in the case that there's a redzone.
     565             :   AFI->setHasRedZone(false);
     566             : 
     567             :   // Debug location must be unknown since the first debug location is used
     568             :   // to determine the end of the prologue.
     569       13980 :   DebugLoc DL;
     570             : 
     571             :   // All calls are tail calls in GHC calling conv, and functions have no
     572             :   // prologue/epilogue.
     573       27960 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     574             :     return;
     575             : 
     576       13974 :   int NumBytes = (int)MFI.getStackSize();
     577       13974 :   if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
     578             :     assert(!HasFP && "unexpected function without stack frame but with FP");
     579             : 
     580             :     // All of the stack allocation is for locals.
     581             :     AFI->setLocalStackSize(NumBytes);
     582             : 
     583       12725 :     if (!NumBytes)
     584             :       return;
     585             :     // REDZONE: If the stack size is less than 128 bytes, we don't need
     586             :     // to actually allocate.
     587         223 :     if (canUseRedZone(MF)) {
     588             :       AFI->setHasRedZone(true);
     589             :       ++NumRedZoneFunctions;
     590             :     } else {
     591         220 :       emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
     592             :                       MachineInstr::FrameSetup);
     593             : 
     594             :       // Label used to tie together the PROLOG_LABEL and the MachineMoves.
     595         220 :       MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
     596             :       // Encode the stack size of the leaf function.
     597             :       unsigned CFIIndex = MF.addFrameInst(
     598         220 :           MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
     599         660 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     600             :           .addCFIIndex(CFIIndex)
     601             :           .setMIFlags(MachineInstr::FrameSetup);
     602             :     }
     603             :     return;
     604             :   }
     605             : 
     606             :   bool IsWin64 =
     607        1249 :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
     608        1205 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
     609             : 
     610        1249 :   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
     611             :   // All of the remaining stack allocations are for locals.
     612        1249 :   AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
     613             : 
     614        1249 :   bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
     615        1249 :   if (CombineSPBump) {
     616         453 :     emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -NumBytes, TII,
     617             :                     MachineInstr::FrameSetup);
     618             :     NumBytes = 0;
     619         796 :   } else if (PrologueSaveSize != 0) {
     620         796 :     MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(MBB, MBBI, DL, TII,
     621         796 :                                                      -PrologueSaveSize);
     622         796 :     NumBytes -= PrologueSaveSize;
     623             :   }
     624             :   assert(NumBytes >= 0 && "Negative stack allocation size!?");
     625             : 
     626             :   // Move past the saves of the callee-saved registers, fixing up the offsets
     627             :   // and pre-inc if we decided to combine the callee-save and local stack
     628             :   // pointer bump above.
     629             :   MachineBasicBlock::iterator End = MBB.end();
     630        3267 :   while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
     631        2018 :     if (CombineSPBump)
     632         805 :       fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize());
     633             :     ++MBBI;
     634             :   }
     635        1249 :   if (HasFP) {
     636             :     // Only set up FP if we actually need to. Frame pointer is fp =
     637             :     // sp - fixedobject - 16.
     638         291 :     int FPOffset = AFI->getCalleeSavedStackSize() - 16;
     639         291 :     if (CombineSPBump)
     640         102 :       FPOffset += AFI->getLocalStackSize();
     641             : 
     642             :     // Issue    sub fp, sp, FPOffset or
     643             :     //          mov fp,sp          when FPOffset is zero.
     644             :     // Note: All stores of callee-saved registers are marked as "FrameSetup".
     645             :     // This code marks the instruction(s) that set the FP also.
     646         291 :     emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, FPOffset, TII,
     647             :                     MachineInstr::FrameSetup);
     648             :   }
     649             : 
     650        1249 :   if (windowsRequiresStackProbe(MF, NumBytes)) {
     651           2 :     uint32_t NumWords = NumBytes >> 4;
     652             : 
     653           6 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
     654           2 :         .addImm(NumWords)
     655             :         .setMIFlags(MachineInstr::FrameSetup);
     656             : 
     657           2 :     switch (MF.getTarget().getCodeModel()) {
     658           1 :     case CodeModel::Small:
     659             :     case CodeModel::Medium:
     660             :     case CodeModel::Kernel:
     661           3 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
     662             :           .addExternalSymbol("__chkstk")
     663           1 :           .addReg(AArch64::X15, RegState::Implicit)
     664             :           .setMIFlags(MachineInstr::FrameSetup);
     665           1 :       break;
     666           1 :     case CodeModel::Large:
     667           3 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
     668           1 :           .addReg(AArch64::X16, RegState::Define)
     669             :           .addExternalSymbol("__chkstk")
     670             :           .addExternalSymbol("__chkstk")
     671             :           .setMIFlags(MachineInstr::FrameSetup);
     672             : 
     673           3 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
     674           1 :           .addReg(AArch64::X16, RegState::Kill)
     675           1 :           .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
     676             :           .setMIFlags(MachineInstr::FrameSetup);
     677           1 :       break;
     678             :     }
     679             : 
     680           6 :     BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
     681           2 :         .addReg(AArch64::SP, RegState::Kill)
     682           2 :         .addReg(AArch64::X15, RegState::Kill)
     683           2 :         .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
     684             :         .setMIFlags(MachineInstr::FrameSetup);
     685             :     NumBytes = 0;
     686             :   }
     687             : 
     688             :   // Allocate space for the rest of the frame.
     689        1247 :   if (NumBytes) {
     690          53 :     const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
     691             :     unsigned scratchSPReg = AArch64::SP;
     692             : 
     693          53 :     if (NeedsRealignment) {
     694          20 :       scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
     695             :       assert(scratchSPReg != AArch64::NoRegister);
     696             :     }
     697             : 
     698             :     // If we're a leaf function, try using the red zone.
     699          53 :     if (!canUseRedZone(MF))
     700             :       // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
     701             :       // the correct value here, as NumBytes also includes padding bytes,
     702             :       // which shouldn't be counted here.
     703          52 :       emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP, -NumBytes, TII,
     704             :                       MachineInstr::FrameSetup);
     705             : 
     706          53 :     if (NeedsRealignment) {
     707          20 :       const unsigned Alignment = MFI.getMaxAlignment();
     708          20 :       const unsigned NrBitsToZero = countTrailingZeros(Alignment);
     709             :       assert(NrBitsToZero > 1);
     710             :       assert(scratchSPReg != AArch64::SP);
     711             : 
     712             :       // SUB X9, SP, NumBytes
     713             :       //   -- X9 is temporary register, so shouldn't contain any live data here,
     714             :       //   -- free to use. This is already produced by emitFrameOffset above.
     715             :       // AND SP, X9, 0b11111...0000
     716             :       // The logical immediates have a non-trivial encoding. The following
     717             :       // formula computes the encoded immediate with all ones but
     718             :       // NrBitsToZero zero bits as least significant bits.
     719          20 :       uint32_t andMaskEncoded = (1 << 12)                         // = N
     720          20 :                                 | ((64 - NrBitsToZero) << 6)      // immr
     721          20 :                                 | ((64 - NrBitsToZero - 1) << 0); // imms
     722             : 
     723          60 :       BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
     724          20 :           .addReg(scratchSPReg, RegState::Kill)
     725          20 :           .addImm(andMaskEncoded);
     726             :       AFI->setStackRealigned(true);
     727             :     }
     728             :   }
     729             : 
     730             :   // If we need a base pointer, set it up here. It's whatever the value of the
     731             :   // stack pointer is at this point. Any variable size objects will be allocated
     732             :   // after this, so we can still use the base pointer to reference locals.
     733             :   //
     734             :   // FIXME: Clarify FrameSetup flags here.
     735             :   // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
     736             :   // needed.
     737        1249 :   if (RegInfo->hasBasePointer(MF)) {
     738          15 :     TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
     739          15 :                      false);
     740             :   }
     741             : 
     742        1249 :   if (needsFrameMoves) {
     743         963 :     const DataLayout &TD = MF.getDataLayout();
     744         963 :     const int StackGrowth = -TD.getPointerSize(0);
     745         963 :     unsigned FramePtr = RegInfo->getFrameRegister(MF);
     746             :     // An example of the prologue:
     747             :     //
     748             :     //     .globl __foo
     749             :     //     .align 2
     750             :     //  __foo:
     751             :     // Ltmp0:
     752             :     //     .cfi_startproc
     753             :     //     .cfi_personality 155, ___gxx_personality_v0
     754             :     // Leh_func_begin:
     755             :     //     .cfi_lsda 16, Lexception33
     756             :     //
     757             :     //     stp  xa,bx, [sp, -#offset]!
     758             :     //     ...
     759             :     //     stp  x28, x27, [sp, #offset-32]
     760             :     //     stp  fp, lr, [sp, #offset-16]
     761             :     //     add  fp, sp, #offset - 16
     762             :     //     sub  sp, sp, #1360
     763             :     //
     764             :     // The Stack:
     765             :     //       +-------------------------------------------+
     766             :     // 10000 | ........ | ........ | ........ | ........ |
     767             :     // 10004 | ........ | ........ | ........ | ........ |
     768             :     //       +-------------------------------------------+
     769             :     // 10008 | ........ | ........ | ........ | ........ |
     770             :     // 1000c | ........ | ........ | ........ | ........ |
     771             :     //       +===========================================+
     772             :     // 10010 |                X28 Register               |
     773             :     // 10014 |                X28 Register               |
     774             :     //       +-------------------------------------------+
     775             :     // 10018 |                X27 Register               |
     776             :     // 1001c |                X27 Register               |
     777             :     //       +===========================================+
     778             :     // 10020 |                Frame Pointer              |
     779             :     // 10024 |                Frame Pointer              |
     780             :     //       +-------------------------------------------+
     781             :     // 10028 |                Link Register              |
     782             :     // 1002c |                Link Register              |
     783             :     //       +===========================================+
     784             :     // 10030 | ........ | ........ | ........ | ........ |
     785             :     // 10034 | ........ | ........ | ........ | ........ |
     786             :     //       +-------------------------------------------+
     787             :     // 10038 | ........ | ........ | ........ | ........ |
     788             :     // 1003c | ........ | ........ | ........ | ........ |
     789             :     //       +-------------------------------------------+
     790             :     //
     791             :     //     [sp] = 10030        ::    >>initial value<<
     792             :     //     sp = 10020          ::  stp fp, lr, [sp, #-16]!
     793             :     //     fp = sp == 10020    ::  mov fp, sp
     794             :     //     [sp] == 10020       ::  stp x28, x27, [sp, #-16]!
     795             :     //     sp == 10010         ::    >>final value<<
     796             :     //
     797             :     // The frame pointer (w29) points to address 10020. If we use an offset of
     798             :     // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
     799             :     // for w27, and -32 for w28:
     800             :     //
     801             :     //  Ltmp1:
     802             :     //     .cfi_def_cfa w29, 16
     803             :     //  Ltmp2:
     804             :     //     .cfi_offset w30, -8
     805             :     //  Ltmp3:
     806             :     //     .cfi_offset w29, -16
     807             :     //  Ltmp4:
     808             :     //     .cfi_offset w27, -24
     809             :     //  Ltmp5:
     810             :     //     .cfi_offset w28, -32
     811             : 
     812         963 :     if (HasFP) {
     813             :       // Define the current CFA rule to use the provided FP.
     814         184 :       unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
     815         368 :       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
     816         184 :           nullptr, Reg, 2 * StackGrowth - FixedObject));
     817         552 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     818             :           .addCFIIndex(CFIIndex)
     819             :           .setMIFlags(MachineInstr::FrameSetup);
     820             :     } else {
     821             :       // Encode the stack size of the leaf function.
     822             :       unsigned CFIIndex = MF.addFrameInst(
     823        1558 :           MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize()));
     824        2337 :       BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
     825             :           .addCFIIndex(CFIIndex)
     826             :           .setMIFlags(MachineInstr::FrameSetup);
     827             :     }
     828             : 
     829             :     // Now emit the moves for whatever callee saved regs we have (including FP,
     830             :     // LR if those are saved).
     831         963 :     emitCalleeSavedFrameMoves(MBB, MBBI);
     832             :   }
     833             : }
     834             : 
     835       14217 : void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
     836             :                                         MachineBasicBlock &MBB) const {
     837       14217 :   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
     838       14217 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     839       14217 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
     840             :   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
     841       14217 :   DebugLoc DL;
     842             :   bool IsTailCallReturn = false;
     843       14217 :   if (MBB.end() != MBBI) {
     844             :     DL = MBBI->getDebugLoc();
     845       14217 :     unsigned RetOpcode = MBBI->getOpcode();
     846       14217 :     IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
     847             :       RetOpcode == AArch64::TCRETURNri;
     848             :   }
     849       14217 :   int NumBytes = MFI.getStackSize();
     850       14217 :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
     851             : 
     852             :   // All calls are tail calls in GHC calling conv, and functions have no
     853             :   // prologue/epilogue.
     854       28434 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     855             :     return;
     856             : 
     857             :   // Initial and residual are named for consistency with the prologue. Note that
     858             :   // in the epilogue, the residual adjustment is executed first.
     859             :   uint64_t ArgumentPopSize = 0;
     860       14211 :   if (IsTailCallReturn) {
     861         198 :     MachineOperand &StackAdjust = MBBI->getOperand(1);
     862             : 
     863             :     // For a tail-call in a callee-pops-arguments environment, some or all of
     864             :     // the stack may actually be in use for the call's arguments, this is
     865             :     // calculated during LowerCall and consumed here...
     866         198 :     ArgumentPopSize = StackAdjust.getImm();
     867             :   } else {
     868             :     // ... otherwise the amount to pop is *all* of the argument space,
     869             :     // conveniently stored in the MachineFunctionInfo by
     870             :     // LowerFormalArguments. This will, of course, be zero for the C calling
     871             :     // convention.
     872       14013 :     ArgumentPopSize = AFI->getArgumentStackToRestore();
     873             :   }
     874             : 
     875             :   // The stack frame should be like below,
     876             :   //
     877             :   //      ----------------------                     ---
     878             :   //      |                    |                      |
     879             :   //      | BytesInStackArgArea|              CalleeArgStackSize
     880             :   //      | (NumReusableBytes) |                (of tail call)
     881             :   //      |                    |                     ---
     882             :   //      |                    |                      |
     883             :   //      ---------------------|        ---           |
     884             :   //      |                    |         |            |
     885             :   //      |   CalleeSavedReg   |         |            |
     886             :   //      | (CalleeSavedStackSize)|      |            |
     887             :   //      |                    |         |            |
     888             :   //      ---------------------|         |         NumBytes
     889             :   //      |                    |     StackSize  (StackAdjustUp)
     890             :   //      |   LocalStackSize   |         |            |
     891             :   //      | (covering callee   |         |            |
     892             :   //      |       args)        |         |            |
     893             :   //      |                    |         |            |
     894             :   //      ----------------------        ---          ---
     895             :   //
     896             :   // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
     897             :   //             = StackSize + ArgumentPopSize
     898             :   //
     899             :   // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
     900             :   // it as the 2nd argument of AArch64ISD::TC_RETURN.
     901             : 
     902             :   bool IsWin64 =
     903             :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
     904       14175 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
     905             : 
     906             :   uint64_t AfterCSRPopSize = ArgumentPopSize;
     907       14211 :   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
     908       14211 :   bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
     909             :   // Assume we can't combine the last pop with the sp restore.
     910             : 
     911       14211 :   if (!CombineSPBump && PrologueSaveSize != 0) {
     912         804 :     MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
     913             :     // Converting the last ldp to a post-index ldp is valid only if the last
     914             :     // ldp's offset is 0.
     915         804 :     const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
     916             :     // If the offset is 0, convert it to a post-index ldp.
     917         804 :     if (OffsetOp.getImm() == 0) {
     918         798 :       convertCalleeSaveRestoreToSPPrePostIncDec(MBB, Pop, DL, TII,
     919         798 :                                                 PrologueSaveSize);
     920             :     } else {
     921             :       // If not, make sure to emit an add after the last ldp.
     922             :       // We're doing this by transfering the size to be restored from the
     923             :       // adjustment *before* the CSR pops to the adjustment *after* the CSR
     924             :       // pops.
     925           6 :       AfterCSRPopSize += PrologueSaveSize;
     926             :     }
     927             :   }
     928             : 
     929             :   // Move past the restores of the callee-saved registers.
     930             :   // If we plan on combining the sp bump of the local stack size and the callee
     931             :   // save stack size, we might need to adjust the CSR save and restore offsets.
     932       14211 :   MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
     933             :   MachineBasicBlock::iterator Begin = MBB.begin();
     934       16244 :   while (LastPopI != Begin) {
     935             :     --LastPopI;
     936       15740 :     if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
     937             :       ++LastPopI;
     938             :       break;
     939        2033 :     } else if (CombineSPBump)
     940         815 :       fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize());
     941             :   }
     942             : 
     943             :   // If there is a single SP update, insert it before the ret and we're done.
     944       14211 :   if (CombineSPBump) {
     945         684 :     emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
     946         684 :                     NumBytes + AfterCSRPopSize, TII,
     947             :                     MachineInstr::FrameDestroy);
     948         684 :     return;
     949             :   }
     950             : 
     951       13527 :   NumBytes -= PrologueSaveSize;
     952             :   assert(NumBytes >= 0 && "Negative stack allocation size!?");
     953             : 
     954       13527 :   if (!hasFP(MF)) {
     955       13340 :     bool RedZone = canUseRedZone(MF);
     956             :     // If this was a redzone leaf function, we don't need to restore the
     957             :     // stack pointer (but we may need to pop stack args for fastcc).
     958       13340 :     if (RedZone && AfterCSRPopSize == 0)
     959             :       return;
     960             : 
     961       13287 :     bool NoCalleeSaveRestore = PrologueSaveSize == 0;
     962       13287 :     int StackRestoreBytes = RedZone ? 0 : NumBytes;
     963       13287 :     if (NoCalleeSaveRestore)
     964       12670 :       StackRestoreBytes += AfterCSRPopSize;
     965             : 
     966             :     // If we were able to combine the local stack pop with the argument pop,
     967             :     // then we're done.
     968       13287 :     bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
     969             : 
     970             :     // If we're done after this, make sure to help the load store optimizer.
     971       13287 :     if (Done)
     972       13278 :       adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
     973             : 
     974       13287 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
     975             :                     StackRestoreBytes, TII, MachineInstr::FrameDestroy);
     976       13287 :     if (Done)
     977             :       return;
     978             : 
     979             :     NumBytes = 0;
     980             :   }
     981             : 
     982             :   // Restore the original stack pointer.
     983             :   // FIXME: Rather than doing the math here, we should instead just use
     984             :   // non-post-indexed loads for the restores if we aren't actually going to
     985             :   // be able to save any instructions.
     986         196 :   if (MFI.hasVarSizedObjects() || AFI->isStackRealigned())
     987          54 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
     988          54 :                     -AFI->getCalleeSavedStackSize() + 16, TII,
     989             :                     MachineInstr::FrameDestroy);
     990         142 :   else if (NumBytes)
     991           7 :     emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, NumBytes, TII,
     992             :                     MachineInstr::FrameDestroy);
     993             : 
     994             :   // This must be placed after the callee-save restore code because that code
     995             :   // assumes the SP is at the same location as it was after the callee-save save
     996             :   // code in the prologue.
     997         196 :   if (AfterCSRPopSize) {
     998             :     // Find an insertion point for the first ldp so that it goes before the
     999             :     // shadow call stack epilog instruction. This ensures that the restore of
    1000             :     // lr from x18 is placed after the restore from sp.
    1001           9 :     auto FirstSPPopI = MBB.getFirstTerminator();
    1002          11 :     while (FirstSPPopI != Begin) {
    1003             :       auto Prev = std::prev(FirstSPPopI);
    1004          24 :       if (Prev->getOpcode() != AArch64::LDRXpre ||
    1005           2 :           Prev->getOperand(0).getReg() == AArch64::SP)
    1006             :         break;
    1007             :       FirstSPPopI = Prev;
    1008             :     }
    1009             : 
    1010             :     adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
    1011             : 
    1012           9 :     emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
    1013             :                     AfterCSRPopSize, TII, MachineInstr::FrameDestroy);
    1014             :   }
    1015             : }
    1016             : 
    1017             : /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
    1018             : /// debug info.  It's the same as what we use for resolving the code-gen
    1019             : /// references for now.  FIXME: This can go wrong when references are
    1020             : /// SP-relative and simple call frames aren't used.
    1021          12 : int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
    1022             :                                                  int FI,
    1023             :                                                  unsigned &FrameReg) const {
    1024          12 :   return resolveFrameIndexReference(MF, FI, FrameReg);
    1025             : }
    1026             : 
    1027        3298 : int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
    1028             :                                                      int FI, unsigned &FrameReg,
    1029             :                                                      bool PreferFP) const {
    1030        3298 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
    1031             :   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
    1032        3298 :       MF.getSubtarget().getRegisterInfo());
    1033             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1034        3298 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
    1035             :   bool IsWin64 =
    1036        3298 :       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
    1037        2946 :   unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
    1038        3298 :   int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16;
    1039        3298 :   int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
    1040             :   bool isFixed = MFI.isFixedObjectIndex(FI);
    1041        2926 :   bool isCSR = !isFixed && MFI.getObjectOffset(FI) >=
    1042        2926 :                                -((int)AFI->getCalleeSavedStackSize());
    1043             : 
    1044             :   // Use frame pointer to reference fixed objects. Use it for locals if
    1045             :   // there are VLAs or a dynamically realigned SP (and thus the SP isn't
    1046             :   // reliable as a base). Make sure useFPForScavengingIndex() does the
    1047             :   // right thing for the emergency spill slot.
    1048             :   bool UseFP = false;
    1049        3298 :   if (AFI->hasStackFrame()) {
    1050             :     // Note: Keeping the following as multiple 'if' statements rather than
    1051             :     // merging to a single expression for readability.
    1052             :     //
    1053             :     // Argument access should always use the FP.
    1054        2370 :     if (isFixed) {
    1055         194 :       UseFP = hasFP(MF);
    1056        2176 :     } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
    1057             :       // References to the CSR area must use FP if we're re-aligning the stack
    1058             :       // since the dynamically-sized alignment padding is between the SP/BP and
    1059             :       // the CSR area.
    1060             :       assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
    1061             :       UseFP = true;
    1062        2175 :     } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
    1063             :       // If the FPOffset is negative, we have to keep in mind that the
    1064             :       // available offset range for negative offsets is smaller than for
    1065             :       // positive ones. If an offset is
    1066             :       // available via the FP and the SP, use whichever is closest.
    1067         476 :       bool FPOffsetFits = FPOffset >= -256;
    1068         476 :       PreferFP |= Offset > -FPOffset;
    1069             : 
    1070         476 :       if (MFI.hasVarSizedObjects()) {
    1071             :         // If we have variable sized objects, we can use either FP or BP, as the
    1072             :         // SP offset is unknown. We can use the base pointer if we have one and
    1073             :         // FP is not preferred. If not, we're stuck with using FP.
    1074          46 :         bool CanUseBP = RegInfo->hasBasePointer(MF);
    1075          46 :         if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
    1076             :           UseFP = PreferFP;
    1077          40 :         else if (!CanUseBP) // Can't use BP. Forced to use FP.
    1078             :           UseFP = true;
    1079             :         // else we can use BP and FP, but the offset from FP won't fit.
    1080             :         // That will make us scavenge registers which we can probably avoid by
    1081             :         // using BP. If it won't fit for BP either, we'll scavenge anyway.
    1082         430 :       } else if (FPOffset >= 0) {
    1083             :         // Use SP or FP, whichever gives us the best chance of the offset
    1084             :         // being in range for direct access. If the FPOffset is positive,
    1085             :         // that'll always be best, as the SP will be even further away.
    1086             :         UseFP = true;
    1087             :       } else {
    1088             :         // We have the choice between FP and (SP or BP).
    1089         430 :         if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
    1090             :           UseFP = true;
    1091             :       }
    1092             :     }
    1093             :   }
    1094             : 
    1095             :   assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
    1096             :          "In the presence of dynamic stack pointer realignment, "
    1097             :          "non-argument/CSR objects cannot be accessed through the frame pointer");
    1098             : 
    1099         200 :   if (UseFP) {
    1100         357 :     FrameReg = RegInfo->getFrameRegister(MF);
    1101         357 :     return FPOffset;
    1102             :   }
    1103             : 
    1104             :   // Use the base pointer if we have one.
    1105        2941 :   if (RegInfo->hasBasePointer(MF))
    1106          11 :     FrameReg = RegInfo->getBaseRegister();
    1107             :   else {
    1108             :     assert(!MFI.hasVarSizedObjects() &&
    1109             :            "Can't use SP when we have var sized objects.");
    1110        2930 :     FrameReg = AArch64::SP;
    1111             :     // If we're using the red zone for this function, the SP won't actually
    1112             :     // be adjusted, so the offsets will be negative. They're also all
    1113             :     // within range of the signed 9-bit immediate instructions.
    1114        2930 :     if (canUseRedZone(MF))
    1115          23 :       Offset -= AFI->getLocalStackSize();
    1116             :   }
    1117             : 
    1118             :   return Offset;
    1119             : }
    1120             : 
    1121             : static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
    1122             :   // Do not set a kill flag on values that are also marked as live-in. This
    1123             :   // happens with the @llvm-returnaddress intrinsic and with arguments passed in
    1124             :   // callee saved registers.
    1125             :   // Omitting the kill flags is conservatively correct even if the live-in
    1126             :   // is not used after all.
    1127        3355 :   bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
    1128             :   return getKillRegState(!IsLiveIn);
    1129             : }
    1130             : 
    1131      287092 : static bool produceCompactUnwindFrame(MachineFunction &MF) {
    1132      287092 :   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
    1133      287092 :   AttributeList Attrs = MF.getFunction().getAttributes();
    1134      339542 :   return Subtarget.isTargetMachO() &&
    1135      104900 :          !(Subtarget.getTargetLowering()->supportSwiftError() &&
    1136      339542 :            Attrs.hasAttrSomewhere(Attribute::SwiftError));
    1137             : }
    1138             : 
    1139             : namespace {
    1140             : 
    1141             : struct RegPairInfo {
    1142             :   unsigned Reg1 = AArch64::NoRegister;
    1143             :   unsigned Reg2 = AArch64::NoRegister;
    1144             :   int FrameIdx;
    1145             :   int Offset;
    1146             :   bool IsGPR;
    1147             : 
    1148        4044 :   RegPairInfo() = default;
    1149             : 
    1150             :   bool isPaired() const { return Reg2 != AArch64::NoRegister; }
    1151             : };
    1152             : 
    1153             : } // end anonymous namespace
    1154             : 
    1155        2512 : static void computeCalleeSaveRegisterPairs(
    1156             :     MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
    1157             :     const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
    1158             :     bool &NeedShadowCallStackProlog) {
    1159             : 
    1160        2512 :   if (CSI.empty())
    1161             :     return;
    1162             : 
    1163        2512 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1164        2512 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1165             :   CallingConv::ID CC = MF.getFunction().getCallingConv();
    1166        5024 :   unsigned Count = CSI.size();
    1167             :   (void)CC;
    1168             :   // MachO's compact unwind format relies on all registers being stored in
    1169             :   // pairs.
    1170             :   assert((!produceCompactUnwindFrame(MF) ||
    1171             :           CC == CallingConv::PreserveMost ||
    1172             :           (Count & 1) == 0) &&
    1173             :          "Odd number of callee-saved regs to spill!");
    1174        2512 :   int Offset = AFI->getCalleeSavedStackSize();
    1175             : 
    1176       10600 :   for (unsigned i = 0; i < Count; ++i) {
    1177             :     RegPairInfo RPI;
    1178        8088 :     RPI.Reg1 = CSI[i].getReg();
    1179             : 
    1180             :     assert(AArch64::GPR64RegClass.contains(RPI.Reg1) ||
    1181             :            AArch64::FPR64RegClass.contains(RPI.Reg1));
    1182        8088 :     RPI.IsGPR = AArch64::GPR64RegClass.contains(RPI.Reg1);
    1183             : 
    1184             :     // Add the next reg to the pair if it is in the same register class.
    1185        4044 :     if (i + 1 < Count) {
    1186        5670 :       unsigned NextReg = CSI[i + 1].getReg();
    1187        5294 :       if ((RPI.IsGPR && AArch64::GPR64RegClass.contains(NextReg)) ||
    1188         752 :           (!RPI.IsGPR && AArch64::FPR64RegClass.contains(NextReg)))
    1189        2683 :         RPI.Reg2 = NextReg;
    1190             :     }
    1191             : 
    1192             :     // If either of the registers to be saved is the lr register, it means that
    1193             :     // we also need to save lr in the shadow call stack.
    1194        6463 :     if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
    1195        2419 :         MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
    1196           4 :       if (!MF.getSubtarget<AArch64Subtarget>().isX18Reserved())
    1197           0 :         report_fatal_error("Must reserve x18 to use shadow call stack");
    1198           4 :       NeedShadowCallStackProlog = true;
    1199             :     }
    1200             : 
    1201             :     // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
    1202             :     // list to come in sorted by frame index so that we can issue the store
    1203             :     // pair instructions directly. Assert if we see anything otherwise.
    1204             :     //
    1205             :     // The order of the registers in the list is controlled by
    1206             :     // getCalleeSavedRegs(), so they will always be in-order, as well.
    1207             :     assert((!RPI.isPaired() ||
    1208             :             (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
    1209             :            "Out of order callee saved regs!");
    1210             : 
    1211             :     // MachO's compact unwind format relies on all registers being stored in
    1212             :     // adjacent register pairs.
    1213             :     assert((!produceCompactUnwindFrame(MF) ||
    1214             :             CC == CallingConv::PreserveMost ||
    1215             :             (RPI.isPaired() &&
    1216             :              ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
    1217             :               RPI.Reg1 + 1 == RPI.Reg2))) &&
    1218             :            "Callee-save registers not saved as adjacent register pair!");
    1219             : 
    1220        8088 :     RPI.FrameIdx = CSI[i].getFrameIdx();
    1221             : 
    1222        4044 :     if (Count * 8 != AFI->getCalleeSavedStackSize() && !RPI.isPaired()) {
    1223             :       // Round up size of non-pair to pair size if we need to pad the
    1224             :       // callee-save area to ensure 16-byte alignment.
    1225        1105 :       Offset -= 16;
    1226             :       assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
    1227             :       MFI.setObjectAlignment(RPI.FrameIdx, 16);
    1228             :       AFI->setCalleeSaveStackHasFreeSpace(true);
    1229             :     } else
    1230        5878 :       Offset -= RPI.isPaired() ? 16 : 8;
    1231             :     assert(Offset % 8 == 0);
    1232        4044 :     RPI.Offset = Offset / 8;
    1233             :     assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
    1234             :            "Offset out of bounds for LDP/STP immediate");
    1235             : 
    1236        4044 :     RegPairs.push_back(RPI);
    1237        4044 :     if (RPI.isPaired())
    1238             :       ++i;
    1239             :   }
    1240             : }
    1241             : 
    1242        1249 : bool AArch64FrameLowering::spillCalleeSavedRegisters(
    1243             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
    1244             :     const std::vector<CalleeSavedInfo> &CSI,
    1245             :     const TargetRegisterInfo *TRI) const {
    1246        1249 :   MachineFunction &MF = *MBB.getParent();
    1247        1249 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1248        1249 :   DebugLoc DL;
    1249             :   SmallVector<RegPairInfo, 8> RegPairs;
    1250             : 
    1251        1249 :   bool NeedShadowCallStackProlog = false;
    1252        1249 :   computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
    1253             :                                  NeedShadowCallStackProlog);
    1254        1249 :   const MachineRegisterInfo &MRI = MF.getRegInfo();
    1255             : 
    1256        1249 :   if (NeedShadowCallStackProlog) {
    1257             :     // Shadow call stack prolog: str x30, [x18], #8
    1258           6 :     BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
    1259           2 :         .addReg(AArch64::X18, RegState::Define)
    1260           2 :         .addReg(AArch64::LR)
    1261           2 :         .addReg(AArch64::X18)
    1262             :         .addImm(8)
    1263             :         .setMIFlag(MachineInstr::FrameSetup);
    1264             : 
    1265             :     // This instruction also makes x18 live-in to the entry block.
    1266             :     MBB.addLiveIn(AArch64::X18);
    1267             :   }
    1268             : 
    1269        1249 :   for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
    1270             :        ++RPII) {
    1271        2013 :     RegPairInfo RPI = *RPII;
    1272             :     unsigned Reg1 = RPI.Reg1;
    1273             :     unsigned Reg2 = RPI.Reg2;
    1274             :     unsigned StrOpc;
    1275             : 
    1276             :     // Issue sequence of spills for cs regs.  The first spill may be converted
    1277             :     // to a pre-decrement store later by emitPrologue if the callee-save stack
    1278             :     // area allocation can't be combined with the local stack area allocation.
    1279             :     // For example:
    1280             :     //    stp     x22, x21, [sp, #0]     // addImm(+0)
    1281             :     //    stp     x20, x19, [sp, #16]    // addImm(+2)
    1282             :     //    stp     fp, lr, [sp, #32]      // addImm(+4)
    1283             :     // Rationale: This sequence saves uop updates compared to a sequence of
    1284             :     // pre-increment spills like stp xi,xj,[sp,#-16]!
    1285             :     // Note: Similar rationale and sequence for restores in epilog.
    1286        2013 :     if (RPI.IsGPR)
    1287        1756 :       StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
    1288             :     else
    1289         257 :       StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
    1290             :     LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
    1291             :                if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
    1292             :                dbgs() << ") -> fi#(" << RPI.FrameIdx;
    1293             :                if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
    1294             :                dbgs() << ")\n");
    1295             : 
    1296        4026 :     MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
    1297        2013 :     if (!MRI.isReserved(Reg1))
    1298        1998 :       MBB.addLiveIn(Reg1);
    1299        2013 :     if (RPI.isPaired()) {
    1300        1342 :       if (!MRI.isReserved(Reg2))
    1301         870 :         MBB.addLiveIn(Reg2);
    1302        1342 :       MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
    1303             :       MIB.addMemOperand(MF.getMachineMemOperand(
    1304             :           MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
    1305        2684 :           MachineMemOperand::MOStore, 8, 8));
    1306             :     }
    1307        2013 :     MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
    1308        2013 :         .addReg(AArch64::SP)
    1309        2013 :         .addImm(RPI.Offset) // [sp, #offset*8], where factor*8 is implicit
    1310             :         .setMIFlag(MachineInstr::FrameSetup);
    1311             :     MIB.addMemOperand(MF.getMachineMemOperand(
    1312             :         MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
    1313        4026 :         MachineMemOperand::MOStore, 8, 8));
    1314             :   }
    1315        1249 :   return true;
    1316             : }
    1317             : 
    1318        1263 : bool AArch64FrameLowering::restoreCalleeSavedRegisters(
    1319             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
    1320             :     std::vector<CalleeSavedInfo> &CSI,
    1321             :     const TargetRegisterInfo *TRI) const {
    1322        1263 :   MachineFunction &MF = *MBB.getParent();
    1323        1263 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1324        1263 :   DebugLoc DL;
    1325             :   SmallVector<RegPairInfo, 8> RegPairs;
    1326             : 
    1327        1263 :   if (MI != MBB.end())
    1328             :     DL = MI->getDebugLoc();
    1329             : 
    1330        1263 :   bool NeedShadowCallStackProlog = false;
    1331        1263 :   computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
    1332             :                                  NeedShadowCallStackProlog);
    1333             : 
    1334        2031 :   auto EmitMI = [&](const RegPairInfo &RPI) {
    1335        2031 :     unsigned Reg1 = RPI.Reg1;
    1336        2031 :     unsigned Reg2 = RPI.Reg2;
    1337             : 
    1338             :     // Issue sequence of restores for cs regs. The last restore may be converted
    1339             :     // to a post-increment load later by emitEpilogue if the callee-save stack
    1340             :     // area allocation can't be combined with the local stack area allocation.
    1341             :     // For example:
    1342             :     //    ldp     fp, lr, [sp, #32]       // addImm(+4)
    1343             :     //    ldp     x20, x19, [sp, #16]     // addImm(+2)
    1344             :     //    ldp     x22, x21, [sp, #0]      // addImm(+0)
    1345             :     // Note: see comment in spillCalleeSavedRegisters()
    1346             :     unsigned LdrOpc;
    1347        2031 :     if (RPI.IsGPR)
    1348        1774 :       LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
    1349             :     else
    1350         257 :       LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
    1351             :     LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
    1352             :                if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
    1353             :                dbgs() << ") -> fi#(" << RPI.FrameIdx;
    1354             :                if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
    1355             :                dbgs() << ")\n");
    1356             : 
    1357        4062 :     MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
    1358        2031 :     if (RPI.isPaired()) {
    1359        1341 :       MIB.addReg(Reg2, getDefRegState(true));
    1360        3372 :       MIB.addMemOperand(MF.getMachineMemOperand(
    1361        1341 :           MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
    1362        2682 :           MachineMemOperand::MOLoad, 8, 8));
    1363             :     }
    1364        2031 :     MIB.addReg(Reg1, getDefRegState(true))
    1365        2031 :         .addReg(AArch64::SP)
    1366        2031 :         .addImm(RPI.Offset) // [sp, #offset*8] where the factor*8 is implicit
    1367             :         .setMIFlag(MachineInstr::FrameDestroy);
    1368             :     MIB.addMemOperand(MF.getMachineMemOperand(
    1369        2031 :         MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
    1370        4062 :         MachineMemOperand::MOLoad, 8, 8));
    1371        3294 :   };
    1372             : 
    1373        1263 :   if (ReverseCSRRestoreSeq)
    1374          26 :     for (const RegPairInfo &RPI : reverse(RegPairs))
    1375          20 :       EmitMI(RPI);
    1376             :   else
    1377        5279 :     for (const RegPairInfo &RPI : RegPairs)
    1378        2011 :       EmitMI(RPI);
    1379             : 
    1380        1263 :   if (NeedShadowCallStackProlog) {
    1381             :     // Shadow call stack epilog: ldr x30, [x18, #-8]!
    1382           6 :     BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
    1383           2 :         .addReg(AArch64::X18, RegState::Define)
    1384           2 :         .addReg(AArch64::LR, RegState::Define)
    1385           2 :         .addReg(AArch64::X18)
    1386             :         .addImm(-8)
    1387             :         .setMIFlag(MachineInstr::FrameDestroy);
    1388             :   }
    1389             : 
    1390        1263 :   return true;
    1391             : }
    1392             : 
    1393       14180 : void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
    1394             :                                                 BitVector &SavedRegs,
    1395             :                                                 RegScavenger *RS) const {
    1396             :   // All calls are tail calls in GHC calling conv, and functions have no
    1397             :   // prologue/epilogue.
    1398       28360 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
    1399             :     return;
    1400             : 
    1401       14174 :   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
    1402             :   const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
    1403       14174 :       MF.getSubtarget().getRegisterInfo());
    1404       14174 :   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1405             :   unsigned UnspilledCSGPR = AArch64::NoRegister;
    1406             :   unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
    1407             : 
    1408       14174 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1409       14174 :   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
    1410             : 
    1411       14174 :   unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
    1412       14174 :                                 ? RegInfo->getBaseRegister()
    1413             :                                 : (unsigned)AArch64::NoRegister;
    1414             : 
    1415             :   unsigned SpillEstimate = SavedRegs.count();
    1416      581450 :   for (unsigned i = 0; CSRegs[i]; ++i) {
    1417      283638 :     unsigned Reg = CSRegs[i];
    1418      283638 :     unsigned PairedReg = CSRegs[i ^ 1];
    1419      283638 :     if (Reg == BasePointerReg)
    1420          15 :       SpillEstimate++;
    1421      333992 :     if (produceCompactUnwindFrame(MF) && !SavedRegs.test(PairedReg))
    1422       49177 :       SpillEstimate++;
    1423             :   }
    1424       14174 :   SpillEstimate += 2; // Conservatively include FP+LR in the estimate
    1425       14174 :   unsigned StackEstimate = MFI.estimateStackSize(MF) + 8 * SpillEstimate;
    1426             : 
    1427             :   // The frame record needs to be created by saving the appropriate registers
    1428       14174 :   if (hasFP(MF) || windowsRequiresStackProbe(MF, StackEstimate)) {
    1429             :     SavedRegs.set(AArch64::FP);
    1430             :     SavedRegs.set(AArch64::LR);
    1431             :   }
    1432             : 
    1433             :   unsigned ExtraCSSpill = 0;
    1434             :   // Figure out which callee-saved registers to save/restore.
    1435      581450 :   for (unsigned i = 0; CSRegs[i]; ++i) {
    1436      283638 :     const unsigned Reg = CSRegs[i];
    1437             : 
    1438             :     // Add the base pointer register to SavedRegs if it is callee-save.
    1439      283638 :     if (Reg == BasePointerReg)
    1440             :       SavedRegs.set(Reg);
    1441             : 
    1442             :     bool RegUsed = SavedRegs.test(Reg);
    1443      283638 :     unsigned PairedReg = CSRegs[i ^ 1];
    1444      563857 :     if (!RegUsed) {
    1445      727628 :       if (AArch64::GPR64RegClass.contains(Reg) &&
    1446      167190 :           !RegInfo->isReservedReg(MF, Reg)) {
    1447             :         UnspilledCSGPR = Reg;
    1448             :         UnspilledCSGPRPaired = PairedReg;
    1449             :       }
    1450      280219 :       continue;
    1451             :     }
    1452             : 
    1453             :     // MachO's compact unwind format relies on all registers being stored in
    1454             :     // pairs.
    1455             :     // FIXME: the usual format is actually better if unwinding isn't needed.
    1456        5010 :     if (produceCompactUnwindFrame(MF) && !SavedRegs.test(PairedReg)) {
    1457             :       SavedRegs.set(PairedReg);
    1458         758 :       if (AArch64::GPR64RegClass.contains(PairedReg) &&
    1459         242 :           !RegInfo->isReservedReg(MF, PairedReg))
    1460             :         ExtraCSSpill = PairedReg;
    1461             :     }
    1462             :   }
    1463             : 
    1464             :   LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nUsed CSRs:";
    1465             :              for (unsigned Reg
    1466             :                   : SavedRegs.set_bits()) dbgs()
    1467             :              << ' ' << printReg(Reg, RegInfo);
    1468             :              dbgs() << "\n";);
    1469             : 
    1470             :   // If any callee-saved registers are used, the frame cannot be eliminated.
    1471             :   unsigned NumRegsSpilled = SavedRegs.count();
    1472             :   bool CanEliminateFrame = NumRegsSpilled == 0;
    1473             : 
    1474             :   // The CSR spill slots have not been allocated yet, so estimateStackSize
    1475             :   // won't include them.
    1476       14174 :   unsigned CFSize = MFI.estimateStackSize(MF) + 8 * NumRegsSpilled;
    1477             :   LLVM_DEBUG(dbgs() << "Estimated stack frame size: " << CFSize << " bytes.\n");
    1478       14174 :   unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
    1479       14174 :   bool BigStack = (CFSize > EstimatedStackSizeLimit);
    1480       14174 :   if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
    1481             :     AFI->setHasStackFrame(true);
    1482             : 
    1483             :   // Estimate if we might need to scavenge a register at some point in order
    1484             :   // to materialize a stack offset. If so, either spill one additional
    1485             :   // callee-saved register or reserve a special spill slot to facilitate
    1486             :   // register scavenging. If we already spilled an extra callee-saved register
    1487             :   // above to keep the number of spills even, we don't need to do anything else
    1488             :   // here.
    1489       14174 :   if (BigStack) {
    1490          47 :     if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
    1491             :       LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
    1492             :                         << " to get a scratch register.\n");
    1493             :       SavedRegs.set(UnspilledCSGPR);
    1494             :       // MachO's compact unwind format relies on all registers being stored in
    1495             :       // pairs, so if we need to spill one extra for BigStack, then we need to
    1496             :       // store the pair.
    1497          35 :       if (produceCompactUnwindFrame(MF))
    1498             :         SavedRegs.set(UnspilledCSGPRPaired);
    1499             :       ExtraCSSpill = UnspilledCSGPRPaired;
    1500             :       NumRegsSpilled = SavedRegs.count();
    1501             :     }
    1502             : 
    1503             :     // If we didn't find an extra callee-saved register to spill, create
    1504             :     // an emergency spill slot.
    1505          47 :     if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
    1506          12 :       const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1507             :       const TargetRegisterClass &RC = AArch64::GPR64RegClass;
    1508             :       unsigned Size = TRI->getSpillSize(RC);
    1509             :       unsigned Align = TRI->getSpillAlignment(RC);
    1510          12 :       int FI = MFI.CreateStackObject(Size, Align, false);
    1511             :       RS->addScavengingFrameIndex(FI);
    1512             :       LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
    1513             :                         << " as the emergency spill slot.\n");
    1514             :     }
    1515             :   }
    1516             : 
    1517             :   // Round up to register pair alignment to avoid additional SP adjustment
    1518             :   // instructions.
    1519       28348 :   AFI->setCalleeSavedStackSize(alignTo(8 * NumRegsSpilled, 16));
    1520             : }
    1521             : 
    1522         428 : bool AArch64FrameLowering::enableStackSlotScavenging(
    1523             :     const MachineFunction &MF) const {
    1524             :   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
    1525         428 :   return AFI->hasCalleeSaveStackFreeSpace();
    1526      303507 : }

Generated by: LCOV version 1.13