LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMFrameLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 961 995 96.6 %
Date: 2018-09-23 13:06:45 Functions: 29 31 93.5 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- ARMFrameLowering.cpp - ARM Frame Information -----------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the ARM implementation of TargetFrameLowering class.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "ARMFrameLowering.h"
      15             : #include "ARMBaseInstrInfo.h"
      16             : #include "ARMBaseRegisterInfo.h"
      17             : #include "ARMConstantPoolValue.h"
      18             : #include "ARMMachineFunctionInfo.h"
      19             : #include "ARMSubtarget.h"
      20             : #include "MCTargetDesc/ARMAddressingModes.h"
      21             : #include "MCTargetDesc/ARMBaseInfo.h"
      22             : #include "Utils/ARMBaseInfo.h"
      23             : #include "llvm/ADT/BitVector.h"
      24             : #include "llvm/ADT/STLExtras.h"
      25             : #include "llvm/ADT/SmallPtrSet.h"
      26             : #include "llvm/ADT/SmallVector.h"
      27             : #include "llvm/CodeGen/MachineBasicBlock.h"
      28             : #include "llvm/CodeGen/MachineConstantPool.h"
      29             : #include "llvm/CodeGen/MachineFrameInfo.h"
      30             : #include "llvm/CodeGen/MachineFunction.h"
      31             : #include "llvm/CodeGen/MachineInstr.h"
      32             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      33             : #include "llvm/CodeGen/MachineModuleInfo.h"
      34             : #include "llvm/CodeGen/MachineOperand.h"
      35             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      36             : #include "llvm/CodeGen/RegisterScavenging.h"
      37             : #include "llvm/CodeGen/TargetInstrInfo.h"
      38             : #include "llvm/CodeGen/TargetOpcodes.h"
      39             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      40             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      41             : #include "llvm/IR/Attributes.h"
      42             : #include "llvm/IR/CallingConv.h"
      43             : #include "llvm/IR/DebugLoc.h"
      44             : #include "llvm/IR/Function.h"
      45             : #include "llvm/MC/MCContext.h"
      46             : #include "llvm/MC/MCDwarf.h"
      47             : #include "llvm/MC/MCInstrDesc.h"
      48             : #include "llvm/MC/MCRegisterInfo.h"
      49             : #include "llvm/Support/CodeGen.h"
      50             : #include "llvm/Support/CommandLine.h"
      51             : #include "llvm/Support/Compiler.h"
      52             : #include "llvm/Support/Debug.h"
      53             : #include "llvm/Support/ErrorHandling.h"
      54             : #include "llvm/Support/MathExtras.h"
      55             : #include "llvm/Support/raw_ostream.h"
      56             : #include "llvm/Target/TargetMachine.h"
      57             : #include "llvm/Target/TargetOptions.h"
      58             : #include <algorithm>
      59             : #include <cassert>
      60             : #include <cstddef>
      61             : #include <cstdint>
      62             : #include <iterator>
      63             : #include <utility>
      64             : #include <vector>
      65             : 
      66             : #define DEBUG_TYPE "arm-frame-lowering"
      67             : 
      68             : using namespace llvm;
      69             : 
      70             : static cl::opt<bool>
      71             : SpillAlignedNEONRegs("align-neon-spills", cl::Hidden, cl::init(true),
      72             :                      cl::desc("Align ARM NEON spills in prolog and epilog"));
      73             : 
      74             : static MachineBasicBlock::iterator
      75             : skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
      76             :                         unsigned NumAlignedDPRCS2Regs);
      77             : 
      78        5017 : ARMFrameLowering::ARMFrameLowering(const ARMSubtarget &sti)
      79             :     : TargetFrameLowering(StackGrowsDown, sti.getStackAlignment(), 0, 4),
      80       10034 :       STI(sti) {}
      81             : 
      82      356977 : bool ARMFrameLowering::noFramePointerElim(const MachineFunction &MF) const {
      83             :   // iOS always has a FP for backtracking, force other targets to keep their FP
      84             :   // when doing FastISel. The emitted code is currently superior, and in cases
      85             :   // like test-suite's lencod FastISel isn't quite correct when FP is eliminated.
      86      680010 :   return TargetFrameLowering::noFramePointerElim(MF) ||
      87      323033 :          MF.getSubtarget<ARMSubtarget>().useFastISel();
      88             : }
      89             : 
      90             : /// Returns true if the target can safely skip saving callee-saved registers
      91             : /// for noreturn nounwind functions.
      92          14 : bool ARMFrameLowering::enableCalleeSaveSkip(const MachineFunction &MF) const {
      93             :   assert(MF.getFunction().hasFnAttribute(Attribute::NoReturn) &&
      94             :          MF.getFunction().hasFnAttribute(Attribute::NoUnwind) &&
      95             :          !MF.getFunction().hasFnAttribute(Attribute::UWTable));
      96             : 
      97             :   // Frame pointer and link register are not treated as normal CSR, thus we
      98             :   // can always skip CSR saves for nonreturning functions.
      99          14 :   return true;
     100             : }
     101             : 
     102             : /// hasFP - Return true if the specified function should have a dedicated frame
     103             : /// pointer register.  This is true if the function has variable sized allocas
     104             : /// or if frame pointer elimination is disabled.
     105      147593 : bool ARMFrameLowering::hasFP(const MachineFunction &MF) const {
     106      147593 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     107      147593 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     108             : 
     109             :   // ABI-required frame pointer.
     110      147593 :   if (MF.getTarget().Options.DisableFramePointerElim(MF))
     111             :     return true;
     112             : 
     113             :   // Frame pointer required for use within this function.
     114      124686 :   return (RegInfo->needsStackRealignment(MF) ||
     115      124686 :           MFI.hasVarSizedObjects() ||
     116      121397 :           MFI.isFrameAddressTaken());
     117             : }
     118             : 
     119             : /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
     120             : /// not required, we reserve argument space for call sites in the function
     121             : /// immediately on entry to the current function.  This eliminates the need for
     122             : /// add/sub sp brackets around call sites.  Returns true if the call frame is
     123             : /// included as part of the stack frame.
     124       62314 : bool ARMFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
     125       62314 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     126             :   unsigned CFSize = MFI.getMaxCallFrameSize();
     127             :   // It's not always a good idea to include the call frame as part of the
     128             :   // stack frame. ARM (especially Thumb) has small immediate offset to
     129             :   // address the stack frame. So a large call frame can cause poor codegen
     130             :   // and may even makes it impossible to scavenge a register.
     131       62313 :   if (CFSize >= ((1 << 12) - 1) / 2)  // Half of imm12
     132             :     return false;
     133             : 
     134       61101 :   return !MFI.hasVarSizedObjects();
     135             : }
     136             : 
     137             : /// canSimplifyCallFramePseudos - If there is a reserved call frame, the
     138             : /// call frame pseudos can be simplified.  Unlike most targets, having a FP
     139             : /// is not sufficient here since we still may reference some objects via SP
     140             : /// even when FP is available in Thumb2 mode.
     141             : bool
     142       18687 : ARMFrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
     143       18687 :   return hasReservedCallFrame(MF) || MF.getFrameInfo().hasVarSizedObjects();
     144             : }
     145             : 
     146           0 : static bool isCSRestore(MachineInstr &MI, const ARMBaseInstrInfo &TII,
     147             :                         const MCPhysReg *CSRegs) {
     148             :   // Integer spill area is handled with "pop".
     149           0 :   if (isPopOpcode(MI.getOpcode())) {
     150             :     // The first two operands are predicates. The last two are
     151             :     // imp-def and imp-use of SP. Check everything in between.
     152           0 :     for (int i = 5, e = MI.getNumOperands(); i != e; ++i)
     153           0 :       if (!isCalleeSavedRegister(MI.getOperand(i).getReg(), CSRegs))
     154           0 :         return false;
     155           0 :     return true;
     156             :   }
     157           0 :   if ((MI.getOpcode() == ARM::LDR_POST_IMM ||
     158           0 :        MI.getOpcode() == ARM::LDR_POST_REG ||
     159           0 :        MI.getOpcode() == ARM::t2LDR_POST) &&
     160           0 :       isCalleeSavedRegister(MI.getOperand(0).getReg(), CSRegs) &&
     161           0 :       MI.getOperand(1).getReg() == ARM::SP)
     162           0 :     return true;
     163             : 
     164             :   return false;
     165             : }
     166             : 
     167        3742 : static void emitRegPlusImmediate(
     168             :     bool isARM, MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
     169             :     const DebugLoc &dl, const ARMBaseInstrInfo &TII, unsigned DestReg,
     170             :     unsigned SrcReg, int NumBytes, unsigned MIFlags = MachineInstr::NoFlags,
     171             :     ARMCC::CondCodes Pred = ARMCC::AL, unsigned PredReg = 0) {
     172        3742 :   if (isARM)
     173        2409 :     emitARMRegPlusImmediate(MBB, MBBI, dl, DestReg, SrcReg, NumBytes,
     174             :                             Pred, PredReg, TII, MIFlags);
     175             :   else
     176        1333 :     emitT2RegPlusImmediate(MBB, MBBI, dl, DestReg, SrcReg, NumBytes,
     177             :                            Pred, PredReg, TII, MIFlags);
     178        3742 : }
     179             : 
     180             : static void emitSPUpdate(bool isARM, MachineBasicBlock &MBB,
     181             :                          MachineBasicBlock::iterator &MBBI, const DebugLoc &dl,
     182             :                          const ARMBaseInstrInfo &TII, int NumBytes,
     183             :                          unsigned MIFlags = MachineInstr::NoFlags,
     184             :                          ARMCC::CondCodes Pred = ARMCC::AL,
     185             :                          unsigned PredReg = 0) {
     186        2776 :   emitRegPlusImmediate(isARM, MBB, MBBI, dl, TII, ARM::SP, ARM::SP, NumBytes,
     187             :                        MIFlags, Pred, PredReg);
     188             : }
     189             : 
     190        1244 : static int sizeOfSPAdjustment(const MachineInstr &MI) {
     191             :   int RegSize;
     192        2488 :   switch (MI.getOpcode()) {
     193             :   case ARM::VSTMDDB_UPD:
     194             :     RegSize = 8;
     195             :     break;
     196         958 :   case ARM::STMDB_UPD:
     197             :   case ARM::t2STMDB_UPD:
     198             :     RegSize = 4;
     199         958 :     break;
     200             :   case ARM::t2STR_PRE:
     201             :   case ARM::STR_PRE_IMM:
     202             :     return 4;
     203           0 :   default:
     204           0 :     llvm_unreachable("Unknown push or pop like instruction");
     205             :   }
     206             : 
     207             :   int count = 0;
     208             :   // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
     209             :   // pred) so the list starts at 4.
     210        5469 :   for (int i = MI.getNumOperands() - 1; i >= 4; --i)
     211        4233 :     count += RegSize;
     212             :   return count;
     213             : }
     214             : 
     215         343 : static bool WindowsRequiresStackProbe(const MachineFunction &MF,
     216             :                                       size_t StackSizeInBytes) {
     217         343 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     218         343 :   const Function &F = MF.getFunction();
     219         686 :   unsigned StackProbeSize = (MFI.getStackProtectorIndex() > 0) ? 4080 : 4096;
     220         343 :   if (F.hasFnAttribute("stack-probe-size"))
     221           8 :     F.getFnAttribute("stack-probe-size")
     222           8 :         .getValueAsString()
     223           4 :         .getAsInteger(0, StackProbeSize);
     224         343 :   return (StackSizeInBytes >= StackProbeSize) &&
     225           8 :          !F.hasFnAttribute("no-stack-arg-probe");
     226             : }
     227             : 
     228             : namespace {
     229             : 
     230       13276 : struct StackAdjustingInsts {
     231             :   struct InstInfo {
     232             :     MachineBasicBlock::iterator I;
     233             :     unsigned SPAdjust;
     234             :     bool BeforeFPSet;
     235             :   };
     236             : 
     237             :   SmallVector<InstInfo, 4> Insts;
     238             : 
     239             :   void addInst(MachineBasicBlock::iterator I, unsigned SPAdjust,
     240             :                bool BeforeFPSet = false) {
     241        5563 :     InstInfo Info = {I, SPAdjust, BeforeFPSet};
     242        3731 :     Insts.push_back(Info);
     243             :   }
     244             : 
     245             :   void addExtraBytes(const MachineBasicBlock::iterator I, unsigned ExtraBytes) {
     246             :     auto Info =
     247          24 :         llvm::find_if(Insts, [&](InstInfo &Info) { return Info.I == I; });
     248             :     assert(Info != Insts.end() && "invalid sp adjusting instruction");
     249          24 :     Info->SPAdjust += ExtraBytes;
     250             :   }
     251             : 
     252       13276 :   void emitDefCFAOffsets(MachineBasicBlock &MBB, const DebugLoc &dl,
     253             :                          const ARMBaseInstrInfo &TII, bool HasFP) {
     254       13276 :     MachineFunction &MF = *MBB.getParent();
     255             :     unsigned CFAOffset = 0;
     256       18160 :     for (auto &Info : Insts) {
     257        5431 :       if (HasFP && !Info.BeforeFPSet)
     258             :         return;
     259             : 
     260        4884 :       CFAOffset -= Info.SPAdjust;
     261             :       unsigned CFIIndex = MF.addFrameInst(
     262        4884 :           MCCFIInstruction::createDefCfaOffset(nullptr, CFAOffset));
     263        4884 :       BuildMI(MBB, std::next(Info.I), dl,
     264        4884 :               TII.get(TargetOpcode::CFI_INSTRUCTION))
     265             :               .addCFIIndex(CFIIndex)
     266             :               .setMIFlags(MachineInstr::FrameSetup);
     267             :     }
     268             :   }
     269             : };
     270             : 
     271             : } // end anonymous namespace
     272             : 
     273             : /// Emit an instruction sequence that will align the address in
     274             : /// register Reg by zero-ing out the lower bits.  For versions of the
     275             : /// architecture that support Neon, this must be done in a single
     276             : /// instruction, since skipAlignedDPRCS2Spills assumes it is done in a
     277             : /// single instruction. That function only gets called when optimizing
     278             : /// spilling of D registers on a core with the Neon instruction set
     279             : /// present.
     280         204 : static void emitAligningInstructions(MachineFunction &MF, ARMFunctionInfo *AFI,
     281             :                                      const TargetInstrInfo &TII,
     282             :                                      MachineBasicBlock &MBB,
     283             :                                      MachineBasicBlock::iterator MBBI,
     284             :                                      const DebugLoc &DL, const unsigned Reg,
     285             :                                      const unsigned Alignment,
     286             :                                      const bool MustBeSingleInstruction) {
     287             :   const ARMSubtarget &AST =
     288         204 :       static_cast<const ARMSubtarget &>(MF.getSubtarget());
     289         204 :   const bool CanUseBFC = AST.hasV6T2Ops() || AST.hasV7Ops();
     290         204 :   const unsigned AlignMask = Alignment - 1;
     291         204 :   const unsigned NrBitsToZero = countTrailingZeros(Alignment);
     292             :   assert(!AFI->isThumb1OnlyFunction() && "Thumb1 not supported");
     293         204 :   if (!AFI->isThumbFunction()) {
     294             :     // if the BFC instruction is available, use that to zero the lower
     295             :     // bits:
     296             :     //   bfc Reg, #0, log2(Alignment)
     297             :     // otherwise use BIC, if the mask to zero the required number of bits
     298             :     // can be encoded in the bic immediate field
     299             :     //   bic Reg, Reg, Alignment-1
     300             :     // otherwise, emit
     301             :     //   lsr Reg, Reg, log2(Alignment)
     302             :     //   lsl Reg, Reg, log2(Alignment)
     303         114 :     if (CanUseBFC) {
     304         198 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::BFC), Reg)
     305          99 :           .addReg(Reg, RegState::Kill)
     306          99 :           .addImm(~AlignMask)
     307          99 :           .add(predOps(ARMCC::AL));
     308          15 :     } else if (AlignMask <= 255) {
     309          26 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::BICri), Reg)
     310          13 :           .addReg(Reg, RegState::Kill)
     311          13 :           .addImm(AlignMask)
     312          13 :           .add(predOps(ARMCC::AL))
     313          13 :           .add(condCodeOp());
     314             :     } else {
     315             :       assert(!MustBeSingleInstruction &&
     316             :              "Shouldn't call emitAligningInstructions demanding a single "
     317             :              "instruction to be emitted for large stack alignment for a target "
     318             :              "without BFC.");
     319           4 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::MOVsi), Reg)
     320           2 :           .addReg(Reg, RegState::Kill)
     321           2 :           .addImm(ARM_AM::getSORegOpc(ARM_AM::lsr, NrBitsToZero))
     322           2 :           .add(predOps(ARMCC::AL))
     323           2 :           .add(condCodeOp());
     324           4 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::MOVsi), Reg)
     325           2 :           .addReg(Reg, RegState::Kill)
     326           2 :           .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, NrBitsToZero))
     327           2 :           .add(predOps(ARMCC::AL))
     328           2 :           .add(condCodeOp());
     329             :     }
     330             :   } else {
     331             :     // Since this is only reached for Thumb-2 targets, the BFC instruction
     332             :     // should always be available.
     333             :     assert(CanUseBFC);
     334         180 :     BuildMI(MBB, MBBI, DL, TII.get(ARM::t2BFC), Reg)
     335          90 :         .addReg(Reg, RegState::Kill)
     336          90 :         .addImm(~AlignMask)
     337          90 :         .add(predOps(ARMCC::AL));
     338             :   }
     339         204 : }
     340             : 
     341             : /// We need the offset of the frame pointer relative to other MachineFrameInfo
     342             : /// offsets which are encoded relative to SP at function begin.
     343             : /// See also emitPrologue() for how the FP is set up.
     344             : /// Unfortunately we cannot determine this value in determineCalleeSaves() yet
     345             : /// as assignCalleeSavedSpillSlots() hasn't run at this point. Instead we use
     346             : /// this to produce a conservative estimate that we check in an assert() later.
     347           0 : static int getMaxFPOffset(const Function &F, const ARMFunctionInfo &AFI) {
     348             :   // This is a conservative estimation: Assume the frame pointer being r7 and
     349             :   // pc("r15") up to r8 getting spilled before (= 8 registers).
     350       14529 :   return -AFI.getArgRegsSaveSize() - (8 * 4);
     351             : }
     352             : 
     353       13278 : void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     354             :                                     MachineBasicBlock &MBB) const {
     355       13278 :   MachineBasicBlock::iterator MBBI = MBB.begin();
     356       13278 :   MachineFrameInfo  &MFI = MF.getFrameInfo();
     357       13278 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     358       13278 :   MachineModuleInfo &MMI = MF.getMMI();
     359             :   MCContext &Context = MMI.getContext();
     360       13278 :   const TargetMachine &TM = MF.getTarget();
     361       13278 :   const MCRegisterInfo *MRI = Context.getRegisterInfo();
     362       13278 :   const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
     363       13278 :   const ARMBaseInstrInfo &TII = *STI.getInstrInfo();
     364             :   assert(!AFI->isThumb1OnlyFunction() &&
     365             :          "This emitPrologue does not support Thumb1!");
     366       13278 :   bool isARM = !AFI->isThumbFunction();
     367       13278 :   unsigned Align = STI.getFrameLowering()->getStackAlignment();
     368       13278 :   unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
     369       13278 :   unsigned NumBytes = MFI.getStackSize();
     370             :   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
     371             : 
     372             :   // Debug location must be unknown since the first debug location is used
     373             :   // to determine the end of the prologue.
     374       13278 :   DebugLoc dl;
     375             : 
     376       13278 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
     377             : 
     378             :   // Determine the sizes of each callee-save spill areas and record which frame
     379             :   // belongs to which callee-save spill areas.
     380             :   unsigned GPRCS1Size = 0, GPRCS2Size = 0, DPRCSSize = 0;
     381             :   int FramePtrSpillFI = 0;
     382             :   int D8SpillFI = 0;
     383             : 
     384             :   // All calls are tail calls in GHC calling conv, and functions have no
     385             :   // prologue/epilogue.
     386       26556 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     387             :     return;
     388             : 
     389             :   StackAdjustingInsts DefCFAOffsetCandidates;
     390       13276 :   bool HasFP = hasFP(MF);
     391             : 
     392             :   // Allocate the vararg register save area.
     393       13276 :   if (ArgRegsSaveSize) {
     394          44 :     emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize,
     395             :                  MachineInstr::FrameSetup);
     396          44 :     DefCFAOffsetCandidates.addInst(std::prev(MBBI), ArgRegsSaveSize, true);
     397             :   }
     398             : 
     399       13276 :   if (!AFI->hasStackFrame() &&
     400        9653 :       (!STI.isTargetWindows() || !WindowsRequiresStackProbe(MF, NumBytes))) {
     401        9653 :     if (NumBytes - ArgRegsSaveSize != 0) {
     402         390 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -(NumBytes - ArgRegsSaveSize),
     403             :                    MachineInstr::FrameSetup);
     404         390 :       DefCFAOffsetCandidates.addInst(std::prev(MBBI),
     405             :                                      NumBytes - ArgRegsSaveSize, true);
     406             :     }
     407        9653 :     DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
     408             :     return;
     409             :   }
     410             : 
     411             :   // Determine spill area sizes.
     412       18837 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     413       11591 :     unsigned Reg = CSI[i].getReg();
     414       11591 :     int FI = CSI[i].getFrameIdx();
     415       11591 :     switch (Reg) {
     416        2180 :     case ARM::R8:
     417             :     case ARM::R9:
     418             :     case ARM::R10:
     419             :     case ARM::R11:
     420             :     case ARM::R12:
     421        2180 :       if (STI.splitFramePushPop(MF)) {
     422         212 :         GPRCS2Size += 4;
     423         212 :         break;
     424             :       }
     425             :       LLVM_FALLTHROUGH;
     426             :     case ARM::R0:
     427             :     case ARM::R1:
     428             :     case ARM::R2:
     429             :     case ARM::R3:
     430             :     case ARM::R4:
     431             :     case ARM::R5:
     432             :     case ARM::R6:
     433             :     case ARM::R7:
     434             :     case ARM::LR:
     435       10055 :       if (Reg == FramePtr)
     436             :         FramePtrSpillFI = FI;
     437       10055 :       GPRCS1Size += 4;
     438       10055 :       break;
     439        1324 :     default:
     440             :       // This is a DPR. Exclude the aligned DPRCS2 spills.
     441        1324 :       if (Reg == ARM::D8)
     442             :         D8SpillFI = FI;
     443        1324 :       if (Reg < ARM::D8 || Reg >= ARM::D8 + AFI->getNumAlignedDPRCS2Regs())
     444        1052 :         DPRCSSize += 8;
     445             :     }
     446             :   }
     447             : 
     448             :   // Move past area 1.
     449             :   MachineBasicBlock::iterator LastPush = MBB.end(), GPRCS1Push, GPRCS2Push;
     450        3623 :   if (GPRCS1Size > 0) {
     451             :     GPRCS1Push = LastPush = MBBI++;
     452             :     DefCFAOffsetCandidates.addInst(LastPush, GPRCS1Size, true);
     453             :   }
     454             : 
     455             :   // Determine starting offsets of spill areas.
     456        3623 :   unsigned GPRCS1Offset = NumBytes - ArgRegsSaveSize - GPRCS1Size;
     457        3623 :   unsigned GPRCS2Offset = GPRCS1Offset - GPRCS2Size;
     458        3710 :   unsigned DPRAlign = DPRCSSize ? std::min(8U, Align) : 4U;
     459        3623 :   unsigned DPRGapSize = (GPRCS1Size + GPRCS2Size + ArgRegsSaveSize) % DPRAlign;
     460        3623 :   unsigned DPRCSOffset = GPRCS2Offset - DPRGapSize - DPRCSSize;
     461             :   int FramePtrOffsetInPush = 0;
     462        3623 :   if (HasFP) {
     463             :     int FPOffset = MFI.getObjectOffset(FramePtrSpillFI);
     464             :     assert(getMaxFPOffset(MF.getFunction(), *AFI) <= FPOffset &&
     465             :            "Max FP estimation is wrong");
     466         966 :     FramePtrOffsetInPush = FPOffset + ArgRegsSaveSize;
     467         966 :     AFI->setFramePtrSpillOffset(MFI.getObjectOffset(FramePtrSpillFI) +
     468             :                                 NumBytes);
     469             :   }
     470             :   AFI->setGPRCalleeSavedArea1Offset(GPRCS1Offset);
     471             :   AFI->setGPRCalleeSavedArea2Offset(GPRCS2Offset);
     472             :   AFI->setDPRCalleeSavedAreaOffset(DPRCSOffset);
     473             : 
     474             :   // Move past area 2.
     475        3623 :   if (GPRCS2Size > 0) {
     476             :     GPRCS2Push = LastPush = MBBI++;
     477             :     DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size);
     478             :   }
     479             : 
     480             :   // Prolog/epilog inserter assumes we correctly align DPRs on the stack, so our
     481             :   // .cfi_offset operations will reflect that.
     482        3623 :   if (DPRGapSize) {
     483             :     assert(DPRGapSize == 4 && "unexpected alignment requirements for DPRs");
     484          29 :     if (LastPush != MBB.end() &&
     485          14 :         tryFoldSPUpdateIntoPushPop(STI, MF, &*LastPush, DPRGapSize))
     486           2 :       DefCFAOffsetCandidates.addExtraBytes(LastPush, DPRGapSize);
     487             :     else {
     488          13 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -DPRGapSize,
     489             :                    MachineInstr::FrameSetup);
     490          13 :       DefCFAOffsetCandidates.addInst(std::prev(MBBI), DPRGapSize);
     491             :     }
     492             :   }
     493             : 
     494             :   // Move past area 3.
     495        3623 :   if (DPRCSSize > 0) {
     496             :     // Since vpush register list cannot have gaps, there may be multiple vpush
     497             :     // instructions in the prologue.
     498         529 :     while (MBBI != MBB.end() && MBBI->getOpcode() == ARM::VSTMDDB_UPD) {
     499         278 :       DefCFAOffsetCandidates.addInst(MBBI, sizeOfSPAdjustment(*MBBI));
     500             :       LastPush = MBBI++;
     501             :     }
     502             :   }
     503             : 
     504             :   // Move past the aligned DPRCS2 area.
     505        3623 :   if (AFI->getNumAlignedDPRCS2Regs() > 0) {
     506          40 :     MBBI = skipAlignedDPRCS2Spills(MBBI, AFI->getNumAlignedDPRCS2Regs());
     507             :     // The code inserted by emitAlignedDPRCS2Spills realigns the stack, and
     508             :     // leaves the stack pointer pointing to the DPRCS2 area.
     509             :     //
     510             :     // Adjust NumBytes to represent the stack slots below the DPRCS2 area.
     511          40 :     NumBytes += MFI.getObjectOffset(D8SpillFI);
     512             :   } else
     513             :     NumBytes = DPRCSOffset;
     514             : 
     515        7246 :   if (STI.isTargetWindows() && WindowsRequiresStackProbe(MF, NumBytes)) {
     516           3 :     uint32_t NumWords = NumBytes >> 2;
     517             : 
     518           3 :     if (NumWords < 65536)
     519           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi16), ARM::R4)
     520           3 :           .addImm(NumWords)
     521             :           .setMIFlags(MachineInstr::FrameSetup)
     522           3 :           .add(predOps(ARMCC::AL));
     523             :     else
     524           0 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi32imm), ARM::R4)
     525           0 :         .addImm(NumWords)
     526             :         .setMIFlags(MachineInstr::FrameSetup);
     527             : 
     528           3 :     switch (TM.getCodeModel()) {
     529             :     case CodeModel::Tiny:
     530             :       llvm_unreachable("Tiny code model not available on ARM.");
     531           1 :     case CodeModel::Small:
     532             :     case CodeModel::Medium:
     533             :     case CodeModel::Kernel:
     534           2 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tBL))
     535           1 :           .add(predOps(ARMCC::AL))
     536             :           .addExternalSymbol("__chkstk")
     537           1 :           .addReg(ARM::R4, RegState::Implicit)
     538             :           .setMIFlags(MachineInstr::FrameSetup);
     539           1 :       break;
     540           2 :     case CodeModel::Large:
     541           4 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi32imm), ARM::R12)
     542             :         .addExternalSymbol("__chkstk")
     543             :         .setMIFlags(MachineInstr::FrameSetup);
     544             : 
     545           4 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tBLXr))
     546           2 :           .add(predOps(ARMCC::AL))
     547           2 :           .addReg(ARM::R12, RegState::Kill)
     548           2 :           .addReg(ARM::R4, RegState::Implicit)
     549             :           .setMIFlags(MachineInstr::FrameSetup);
     550           2 :       break;
     551             :     }
     552             : 
     553           6 :     BuildMI(MBB, MBBI, dl, TII.get(ARM::t2SUBrr), ARM::SP)
     554           3 :         .addReg(ARM::SP, RegState::Kill)
     555           3 :         .addReg(ARM::R4, RegState::Kill)
     556             :         .setMIFlags(MachineInstr::FrameSetup)
     557           3 :         .add(predOps(ARMCC::AL))
     558           3 :         .add(condCodeOp());
     559             :     NumBytes = 0;
     560             :   }
     561             : 
     562        3620 :   if (NumBytes) {
     563             :     // Adjust SP after all the callee-save spills.
     564        2227 :     if (AFI->getNumAlignedDPRCS2Regs() == 0 &&
     565        1098 :         tryFoldSPUpdateIntoPushPop(STI, MF, &*LastPush, NumBytes))
     566          22 :       DefCFAOffsetCandidates.addExtraBytes(LastPush, NumBytes);
     567             :     else {
     568        1107 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -NumBytes,
     569             :                    MachineInstr::FrameSetup);
     570        1107 :       DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes);
     571             :     }
     572             : 
     573        1129 :     if (HasFP && isARM)
     574             :       // Restore from fp only in ARM mode: e.g. sub sp, r7, #24
     575             :       // Note it's not safe to do this in Thumb2 mode because it would have
     576             :       // taken two instructions:
     577             :       // mov sp, r7
     578             :       // sub sp, #24
     579             :       // If an interrupt is taken between the two instructions, then sp is in
     580             :       // an inconsistent state (pointing to the middle of callee-saved area).
     581             :       // The interrupt handler can end up clobbering the registers.
     582             :       AFI->setShouldRestoreSPFromFP(true);
     583             :   }
     584             : 
     585             :   // Set FP to point to the stack slot that contains the previous FP.
     586             :   // For iOS, FP is R7, which has now been stored in spill area 1.
     587             :   // Otherwise, if this is not iOS, all the callee-saved registers go
     588             :   // into spill area 1, including the FP in R11.  In either case, it
     589             :   // is in area one and the adjustment needs to take place just after
     590             :   // that push.
     591        3623 :   if (HasFP) {
     592         966 :     MachineBasicBlock::iterator AfterPush = std::next(GPRCS1Push);
     593         966 :     unsigned PushSize = sizeOfSPAdjustment(*GPRCS1Push);
     594         966 :     emitRegPlusImmediate(!AFI->isThumbFunction(), MBB, AfterPush,
     595             :                          dl, TII, FramePtr, ARM::SP,
     596         966 :                          PushSize + FramePtrOffsetInPush,
     597             :                          MachineInstr::FrameSetup);
     598         966 :     if (FramePtrOffsetInPush + PushSize != 0) {
     599         480 :       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
     600         480 :           nullptr, MRI->getDwarfRegNum(FramePtr, true),
     601         480 :           -(ArgRegsSaveSize - FramePtrOffsetInPush)));
     602         960 :       BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     603             :           .addCFIIndex(CFIIndex)
     604             :           .setMIFlags(MachineInstr::FrameSetup);
     605             :     } else {
     606             :       unsigned CFIIndex =
     607         486 :           MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
     608         486 :               nullptr, MRI->getDwarfRegNum(FramePtr, true)));
     609         972 :       BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     610             :           .addCFIIndex(CFIIndex)
     611             :           .setMIFlags(MachineInstr::FrameSetup);
     612             :     }
     613             :   }
     614             : 
     615             :   // Now that the prologue's actual instructions are finalised, we can insert
     616             :   // the necessary DWARF cf instructions to describe the situation. Start by
     617             :   // recording where each register ended up:
     618        3623 :   if (GPRCS1Size > 0) {
     619        3615 :     MachineBasicBlock::iterator Pos = std::next(GPRCS1Push);
     620             :     int CFIIndex;
     621       15160 :     for (const auto &Entry : CSI) {
     622       11545 :       unsigned Reg = Entry.getReg();
     623       11545 :       int FI = Entry.getFrameIdx();
     624       11545 :       switch (Reg) {
     625        2180 :       case ARM::R8:
     626             :       case ARM::R9:
     627             :       case ARM::R10:
     628             :       case ARM::R11:
     629             :       case ARM::R12:
     630        2180 :         if (STI.splitFramePushPop(MF))
     631             :           break;
     632             :         LLVM_FALLTHROUGH;
     633             :       case ARM::R0:
     634             :       case ARM::R1:
     635             :       case ARM::R2:
     636             :       case ARM::R3:
     637             :       case ARM::R4:
     638             :       case ARM::R5:
     639             :       case ARM::R6:
     640             :       case ARM::R7:
     641             :       case ARM::LR:
     642       10055 :         CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
     643       10055 :             nullptr, MRI->getDwarfRegNum(Reg, true), MFI.getObjectOffset(FI)));
     644       20110 :         BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     645             :             .addCFIIndex(CFIIndex)
     646             :             .setMIFlags(MachineInstr::FrameSetup);
     647       10055 :         break;
     648             :       }
     649             :     }
     650             :   }
     651             : 
     652        3623 :   if (GPRCS2Size > 0) {
     653         116 :     MachineBasicBlock::iterator Pos = std::next(GPRCS2Push);
     654        1229 :     for (const auto &Entry : CSI) {
     655        1113 :       unsigned Reg = Entry.getReg();
     656        1113 :       int FI = Entry.getFrameIdx();
     657        1113 :       switch (Reg) {
     658         212 :       case ARM::R8:
     659             :       case ARM::R9:
     660             :       case ARM::R10:
     661             :       case ARM::R11:
     662             :       case ARM::R12:
     663         212 :         if (STI.splitFramePushPop(MF)) {
     664         212 :           unsigned DwarfReg =  MRI->getDwarfRegNum(Reg, true);
     665             :           unsigned Offset = MFI.getObjectOffset(FI);
     666             :           unsigned CFIIndex = MF.addFrameInst(
     667         212 :               MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     668         424 :           BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     669             :               .addCFIIndex(CFIIndex)
     670             :               .setMIFlags(MachineInstr::FrameSetup);
     671             :         }
     672             :         break;
     673             :       }
     674             :     }
     675             :   }
     676             : 
     677        3623 :   if (DPRCSSize > 0) {
     678             :     // Since vpush register list cannot have gaps, there may be multiple vpush
     679             :     // instructions in the prologue.
     680         251 :     MachineBasicBlock::iterator Pos = std::next(LastPush);
     681        2115 :     for (const auto &Entry : CSI) {
     682        1864 :       unsigned Reg = Entry.getReg();
     683        1864 :       int FI = Entry.getFrameIdx();
     684        1864 :       if ((Reg >= ARM::D0 && Reg <= ARM::D31) &&
     685         927 :           (Reg < ARM::D8 || Reg >= ARM::D8 + AFI->getNumAlignedDPRCS2Regs())) {
     686        1052 :         unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
     687             :         unsigned Offset = MFI.getObjectOffset(FI);
     688             :         unsigned CFIIndex = MF.addFrameInst(
     689        1052 :             MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     690        2104 :         BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     691             :             .addCFIIndex(CFIIndex)
     692             :             .setMIFlags(MachineInstr::FrameSetup);
     693             :       }
     694             :     }
     695             :   }
     696             : 
     697             :   // Now we can emit descriptions of where the canonical frame address was
     698             :   // throughout the process. If we have a frame pointer, it takes over the job
     699             :   // half-way through, so only the first few .cfi_def_cfa_offset instructions
     700             :   // actually get emitted.
     701        3623 :   DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
     702             : 
     703        7246 :   if (STI.isTargetELF() && hasFP(MF))
     704         750 :     MFI.setOffsetAdjustment(MFI.getOffsetAdjustment() -
     705         375 :                             AFI->getFramePtrSpillOffset());
     706             : 
     707             :   AFI->setGPRCalleeSavedArea1Size(GPRCS1Size);
     708             :   AFI->setGPRCalleeSavedArea2Size(GPRCS2Size);
     709             :   AFI->setDPRCalleeSavedGapSize(DPRGapSize);
     710             :   AFI->setDPRCalleeSavedAreaSize(DPRCSSize);
     711             : 
     712             :   // If we need dynamic stack realignment, do it here. Be paranoid and make
     713             :   // sure if we also have VLAs, we have a base pointer for frame access.
     714             :   // If aligned NEON registers were spilled, the stack has already been
     715             :   // realigned.
     716        3623 :   if (!AFI->getNumAlignedDPRCS2Regs() && RegInfo->needsStackRealignment(MF)) {
     717         164 :     unsigned MaxAlign = MFI.getMaxAlignment();
     718             :     assert(!AFI->isThumb1OnlyFunction());
     719         164 :     if (!AFI->isThumbFunction()) {
     720          98 :       emitAligningInstructions(MF, AFI, TII, MBB, MBBI, dl, ARM::SP, MaxAlign,
     721             :                                false);
     722             :     } else {
     723             :       // We cannot use sp as source/dest register here, thus we're using r4 to
     724             :       // perform the calculations. We're emitting the following sequence:
     725             :       // mov r4, sp
     726             :       // -- use emitAligningInstructions to produce best sequence to zero
     727             :       // -- out lower bits in r4
     728             :       // mov sp, r4
     729             :       // FIXME: It will be better just to find spare register here.
     730         132 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::R4)
     731          66 :           .addReg(ARM::SP, RegState::Kill)
     732          66 :           .add(predOps(ARMCC::AL));
     733          66 :       emitAligningInstructions(MF, AFI, TII, MBB, MBBI, dl, ARM::R4, MaxAlign,
     734             :                                false);
     735         132 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     736          66 :           .addReg(ARM::R4, RegState::Kill)
     737          66 :           .add(predOps(ARMCC::AL));
     738             :     }
     739             : 
     740             :     AFI->setShouldRestoreSPFromFP(true);
     741             :   }
     742             : 
     743             :   // If we need a base pointer, set it up here. It's whatever the value
     744             :   // of the stack pointer is at this point. Any variable size objects
     745             :   // will be allocated after this, so we can still use the base pointer
     746             :   // to reference locals.
     747             :   // FIXME: Clarify FrameSetup flags here.
     748        3623 :   if (RegInfo->hasBasePointer(MF)) {
     749           6 :     if (isARM)
     750           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), RegInfo->getBaseRegister())
     751           3 :           .addReg(ARM::SP)
     752           3 :           .add(predOps(ARMCC::AL))
     753           3 :           .add(condCodeOp());
     754             :     else
     755           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), RegInfo->getBaseRegister())
     756           3 :           .addReg(ARM::SP)
     757           3 :           .add(predOps(ARMCC::AL));
     758             :   }
     759             : 
     760             :   // If the frame has variable sized objects then the epilogue must restore
     761             :   // the sp from fp. We can assume there's an FP here since hasFP already
     762             :   // checks for hasVarSizedObjects.
     763        3623 :   if (MFI.hasVarSizedObjects())
     764             :     AFI->setShouldRestoreSPFromFP(true);
     765             : }
     766             : 
     767       13303 : void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
     768             :                                     MachineBasicBlock &MBB) const {
     769       13303 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     770       13303 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     771       13303 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     772             :   const ARMBaseInstrInfo &TII =
     773       13303 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
     774             :   assert(!AFI->isThumb1OnlyFunction() &&
     775             :          "This emitEpilogue does not support Thumb1!");
     776       13303 :   bool isARM = !AFI->isThumbFunction();
     777             : 
     778       13303 :   unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
     779       13303 :   int NumBytes = (int)MFI.getStackSize();
     780       13303 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
     781             : 
     782             :   // All calls are tail calls in GHC calling conv, and functions have no
     783             :   // prologue/epilogue.
     784       26606 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     785           2 :     return;
     786             : 
     787             :   // First put ourselves on the first (from top) terminator instructions.
     788       13301 :   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
     789       13301 :   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
     790             : 
     791       13301 :   if (!AFI->hasStackFrame()) {
     792        9731 :     if (NumBytes - ArgRegsSaveSize != 0)
     793         388 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ArgRegsSaveSize);
     794             :   } else {
     795             :     // Unwind MBBI to point to first LDR / VLDRD.
     796        3570 :     const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
     797        3570 :     if (MBBI != MBB.begin()) {
     798             :       do {
     799             :         --MBBI;
     800        4879 :       } while (MBBI != MBB.begin() && isCSRestore(*MBBI, TII, CSRegs));
     801        3427 :       if (!isCSRestore(*MBBI, TII, CSRegs))
     802             :         ++MBBI;
     803             :     }
     804             : 
     805             :     // Move SP to start of FP callee save spill area.
     806        7140 :     NumBytes -= (ArgRegsSaveSize +
     807        7140 :                  AFI->getGPRCalleeSavedArea1Size() +
     808        7140 :                  AFI->getGPRCalleeSavedArea2Size() +
     809        7140 :                  AFI->getDPRCalleeSavedGapSize() +
     810        3570 :                  AFI->getDPRCalleeSavedAreaSize());
     811             : 
     812             :     // Reset SP based on frame pointer only if the stack frame extends beyond
     813             :     // frame pointer stack slot or target is ELF and the function has FP.
     814        3570 :     if (AFI->shouldRestoreSPFromFP()) {
     815         382 :       NumBytes = AFI->getFramePtrSpillOffset() - NumBytes;
     816         382 :       if (NumBytes) {
     817         254 :         if (isARM)
     818         154 :           emitARMRegPlusImmediate(MBB, MBBI, dl, ARM::SP, FramePtr, -NumBytes,
     819             :                                   ARMCC::AL, 0, TII);
     820             :         else {
     821             :           // It's not possible to restore SP from FP in a single instruction.
     822             :           // For iOS, this looks like:
     823             :           // mov sp, r7
     824             :           // sub sp, #24
     825             :           // This is bad, if an interrupt is taken after the mov, sp is in an
     826             :           // inconsistent state.
     827             :           // Use the first callee-saved register as a scratch register.
     828             :           assert(!MFI.getPristineRegs(MF).test(ARM::R4) &&
     829             :                  "No scratch register to restore SP from FP!");
     830         100 :           emitT2RegPlusImmediate(MBB, MBBI, dl, ARM::R4, FramePtr, -NumBytes,
     831             :                                  ARMCC::AL, 0, TII);
     832         200 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     833         100 :               .addReg(ARM::R4)
     834         100 :               .add(predOps(ARMCC::AL));
     835             :         }
     836             :       } else {
     837             :         // Thumb2 or ARM.
     838         128 :         if (isARM)
     839         256 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), ARM::SP)
     840         128 :               .addReg(FramePtr)
     841         128 :               .add(predOps(ARMCC::AL))
     842         128 :               .add(condCodeOp());
     843             :         else
     844           0 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     845           0 :               .addReg(FramePtr)
     846           0 :               .add(predOps(ARMCC::AL));
     847             :       }
     848        3937 :     } else if (NumBytes &&
     849         749 :                !tryFoldSPUpdateIntoPushPop(STI, MF, &*MBBI, NumBytes))
     850         735 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes);
     851             : 
     852             :     // Increment past our save areas.
     853        3570 :     if (MBBI != MBB.end() && AFI->getDPRCalleeSavedAreaSize()) {
     854             :       MBBI++;
     855             :       // Since vpop register list cannot have gaps, there may be multiple vpop
     856             :       // instructions in the epilogue.
     857         287 :       while (MBBI != MBB.end() && MBBI->getOpcode() == ARM::VLDMDIA_UPD)
     858             :         MBBI++;
     859             :     }
     860        3570 :     if (AFI->getDPRCalleeSavedGapSize()) {
     861             :       assert(AFI->getDPRCalleeSavedGapSize() == 4 &&
     862             :              "unexpected DPR alignment gap");
     863          13 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, AFI->getDPRCalleeSavedGapSize());
     864             :     }
     865             : 
     866        3570 :     if (AFI->getGPRCalleeSavedArea2Size()) MBBI++;
     867        3570 :     if (AFI->getGPRCalleeSavedArea1Size()) MBBI++;
     868             :   }
     869             : 
     870       13301 :   if (ArgRegsSaveSize)
     871          44 :     emitSPUpdate(isARM, MBB, MBBI, dl, TII, ArgRegsSaveSize);
     872             : }
     873             : 
     874             : /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
     875             : /// debug info.  It's the same as what we use for resolving the code-gen
     876             : /// references for now.  FIXME: This can go wrong when references are
     877             : /// SP-relative and simple call frames aren't used.
     878             : int
     879          22 : ARMFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
     880             :                                          unsigned &FrameReg) const {
     881          22 :   return ResolveFrameIndexReference(MF, FI, FrameReg, 0);
     882             : }
     883             : 
     884             : int
     885       15095 : ARMFrameLowering::ResolveFrameIndexReference(const MachineFunction &MF,
     886             :                                              int FI, unsigned &FrameReg,
     887             :                                              int SPAdj) const {
     888       15095 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     889             :   const ARMBaseRegisterInfo *RegInfo = static_cast<const ARMBaseRegisterInfo *>(
     890       15095 :       MF.getSubtarget().getRegisterInfo());
     891             :   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     892       15095 :   int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
     893       15095 :   int FPOffset = Offset - AFI->getFramePtrSpillOffset();
     894             :   bool isFixed = MFI.isFixedObjectIndex(FI);
     895             : 
     896       15095 :   FrameReg = ARM::SP;
     897       15095 :   Offset += SPAdj;
     898             : 
     899             :   // SP can move around if there are allocas.  We may also lose track of SP
     900             :   // when emergency spilling inside a non-reserved call frame setup.
     901       15095 :   bool hasMovingSP = !hasReservedCallFrame(MF);
     902             : 
     903             :   // When dynamically realigning the stack, use the frame pointer for
     904             :   // parameters, and the stack/base pointer for locals.
     905       15095 :   if (RegInfo->needsStackRealignment(MF)) {
     906             :     assert(hasFP(MF) && "dynamic stack realignment without a FP!");
     907        7225 :     if (isFixed) {
     908          66 :       FrameReg = RegInfo->getFrameRegister(MF);
     909             :       Offset = FPOffset;
     910        7159 :     } else if (hasMovingSP) {
     911             :       assert(RegInfo->hasBasePointer(MF) &&
     912             :              "VLAs and dynamic stack alignment, but missing base pointer!");
     913         468 :       FrameReg = RegInfo->getBaseRegister();
     914             :     }
     915        7225 :     return Offset;
     916             :   }
     917             : 
     918             :   // If there is a frame pointer, use it when we can.
     919        7870 :   if (hasFP(MF) && AFI->hasStackFrame()) {
     920             :     // Use frame pointer to reference fixed objects. Use it for locals if
     921             :     // there are VLAs (and thus the SP isn't reliable as a base).
     922        1679 :     if (isFixed || (hasMovingSP && !RegInfo->hasBasePointer(MF))) {
     923         272 :       FrameReg = RegInfo->getFrameRegister(MF);
     924         272 :       return FPOffset;
     925        1407 :     } else if (hasMovingSP) {
     926             :       assert(RegInfo->hasBasePointer(MF) && "missing base pointer!");
     927          35 :       if (AFI->isThumb2Function()) {
     928             :         // Try to use the frame pointer if we can, else use the base pointer
     929             :         // since it's available. This is handy for the emergency spill slot, in
     930             :         // particular.
     931           0 :         if (FPOffset >= -255 && FPOffset < 0) {
     932           0 :           FrameReg = RegInfo->getFrameRegister(MF);
     933           0 :           return FPOffset;
     934             :         }
     935             :       }
     936        1372 :     } else if (AFI->isThumbFunction()) {
     937             :       // Prefer SP to base pointer, if the offset is suitably aligned and in
     938             :       // range as the effective range of the immediate offset is bigger when
     939             :       // basing off SP.
     940             :       // Use  add <rd>, sp, #<imm8>
     941             :       //      ldr <rd>, [sp, #<imm8>]
     942         516 :       if (Offset >= 0 && (Offset & 3) == 0 && Offset <= 1020)
     943             :         return Offset;
     944             :       // In Thumb2 mode, the negative offset is very limited. Try to avoid
     945             :       // out of range references. ldr <rt>,[<rn>, #-<imm8>]
     946          24 :       if (AFI->isThumb2Function() && FPOffset >= -255 && FPOffset < 0) {
     947          24 :         FrameReg = RegInfo->getFrameRegister(MF);
     948          24 :         return FPOffset;
     949             :       }
     950         856 :     } else if (Offset > (FPOffset < 0 ? -FPOffset : FPOffset)) {
     951             :       // Otherwise, use SP or FP, whichever is closer to the stack slot.
     952         253 :       FrameReg = RegInfo->getFrameRegister(MF);
     953         253 :       return FPOffset;
     954             :     }
     955             :   }
     956             :   // Use the base pointer if we have one.
     957        6831 :   if (RegInfo->hasBasePointer(MF))
     958          35 :     FrameReg = RegInfo->getBaseRegister();
     959             :   return Offset;
     960             : }
     961             : 
     962       10869 : void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
     963             :                                     MachineBasicBlock::iterator MI,
     964             :                                     const std::vector<CalleeSavedInfo> &CSI,
     965             :                                     unsigned StmOpc, unsigned StrOpc,
     966             :                                     bool NoGap,
     967             :                                     bool(*Func)(unsigned, bool),
     968             :                                     unsigned NumAlignedDPRCS2Regs,
     969             :                                     unsigned MIFlags) const {
     970       10869 :   MachineFunction &MF = *MBB.getParent();
     971       10869 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
     972       10869 :   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
     973             : 
     974       10869 :   DebugLoc DL;
     975             : 
     976             :   using RegAndKill = std::pair<unsigned, bool>;
     977             : 
     978             :   SmallVector<RegAndKill, 4> Regs;
     979       21738 :   unsigned i = CSI.size();
     980       21765 :   while (i != 0) {
     981             :     unsigned LastReg = 0;
     982       45669 :     for (; i != 0; --i) {
     983       34800 :       unsigned Reg = CSI[i-1].getReg();
     984       34800 :       if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue;
     985             : 
     986             :       // D-registers in the aligned area DPRCS2 are NOT spilled here.
     987       11618 :       if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs)
     988             :         continue;
     989             : 
     990       11346 :       const MachineRegisterInfo &MRI = MF.getRegInfo();
     991       11346 :       bool isLiveIn = MRI.isLiveIn(Reg);
     992       11346 :       if (!isLiveIn && !MRI.isReserved(Reg))
     993       10352 :         MBB.addLiveIn(Reg);
     994             :       // If NoGap is true, push consecutive registers and then leave the rest
     995             :       // for other instructions. e.g.
     996             :       // vpush {d8, d10, d11} -> vpush {d8}, vpush {d10, d11}
     997       11346 :       if (NoGap && LastReg && LastReg != Reg-1)
     998             :         break;
     999             :       LastReg = Reg;
    1000             :       // Do not set a kill flag on values that are also marked as live-in. This
    1001             :       // happens with the @llvm-returnaddress intrinsic and with arguments
    1002             :       // passed in callee saved registers.
    1003             :       // Omitting the kill flags is conservatively correct even if the live-in
    1004             :       // is not used after all.
    1005       11319 :       Regs.push_back(std::make_pair(Reg, /*isKill=*/!isLiveIn));
    1006             :     }
    1007             : 
    1008       10896 :     if (Regs.empty())
    1009             :       continue;
    1010             : 
    1011             :     llvm::sort(Regs.begin(), Regs.end(), [&](const RegAndKill &LHS,
    1012             :                                              const RegAndKill &RHS) {
    1013       14886 :       return TRI.getEncodingValue(LHS.first) < TRI.getEncodingValue(RHS.first);
    1014             :     });
    1015             : 
    1016        8018 :     if (Regs.size() > 1 || StrOpc== 0) {
    1017        7448 :       MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StmOpc), ARM::SP)
    1018        3724 :                                     .addReg(ARM::SP)
    1019             :                                     .setMIFlags(MIFlags)
    1020        3724 :                                     .add(predOps(ARMCC::AL));
    1021       14758 :       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
    1022       22077 :         MIB.addReg(Regs[i].first, getKillRegState(Regs[i].second));
    1023         285 :     } else if (Regs.size() == 1) {
    1024         570 :       BuildMI(MBB, MI, DL, TII.get(StrOpc), ARM::SP)
    1025         297 :           .addReg(Regs[0].first, getKillRegState(Regs[0].second))
    1026         285 :           .addReg(ARM::SP)
    1027             :           .setMIFlags(MIFlags)
    1028             :           .addImm(-4)
    1029         285 :           .add(predOps(ARMCC::AL));
    1030             :     }
    1031             :     Regs.clear();
    1032             : 
    1033             :     // Put any subsequent vpush instructions before this one: they will refer to
    1034             :     // higher register numbers so need to be pushed first in order to preserve
    1035             :     // monotonicity.
    1036        4009 :     if (MI != MBB.begin())
    1037             :       --MI;
    1038             :   }
    1039       10869 : }
    1040             : 
    1041       10710 : void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
    1042             :                                    MachineBasicBlock::iterator MI,
    1043             :                                    std::vector<CalleeSavedInfo> &CSI,
    1044             :                                    unsigned LdmOpc, unsigned LdrOpc,
    1045             :                                    bool isVarArg, bool NoGap,
    1046             :                                    bool(*Func)(unsigned, bool),
    1047             :                                    unsigned NumAlignedDPRCS2Regs) const {
    1048       10710 :   MachineFunction &MF = *MBB.getParent();
    1049       10710 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1050       10710 :   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
    1051       10710 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1052       10710 :   DebugLoc DL;
    1053             :   bool isTailCall = false;
    1054             :   bool isInterrupt = false;
    1055             :   bool isTrap = false;
    1056       10710 :   if (MBB.end() != MI) {
    1057             :     DL = MI->getDebugLoc();
    1058       10635 :     unsigned RetOpcode = MI->getOpcode();
    1059       10635 :     isTailCall = (RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNri);
    1060       10635 :     isInterrupt =
    1061       10635 :         RetOpcode == ARM::SUBS_PC_LR || RetOpcode == ARM::t2SUBS_PC_LR;
    1062       10635 :     isTrap =
    1063       10635 :         RetOpcode == ARM::TRAP || RetOpcode == ARM::TRAPNaCl ||
    1064       10635 :         RetOpcode == ARM::tTRAP;
    1065             :   }
    1066             : 
    1067             :   SmallVector<unsigned, 4> Regs;
    1068       21420 :   unsigned i = CSI.size();
    1069       21445 :   while (i != 0) {
    1070             :     unsigned LastReg = 0;
    1071             :     bool DeleteRet = false;
    1072       44812 :     for (; i != 0; --i) {
    1073       34102 :       CalleeSavedInfo &Info = CSI[i-1];
    1074       34102 :       unsigned Reg = Info.getReg();
    1075       34310 :       if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue;
    1076             : 
    1077             :       // The aligned reloads from area DPRCS2 are not inserted here.
    1078       11384 :       if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs)
    1079             :         continue;
    1080             : 
    1081        3524 :       if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt &&
    1082       14574 :           !isTrap && STI.hasV5TOps()) {
    1083        2607 :         if (MBB.succ_empty()) {
    1084        2571 :           Reg = ARM::PC;
    1085             :           // Fold the return instruction into the LDM.
    1086             :           DeleteRet = true;
    1087        2571 :           LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_RET : ARM::LDMIA_RET;
    1088             :           // We 'restore' LR into PC so it is not live out of the return block:
    1089             :           // Clear Restored bit.
    1090             :           Info.setRestored(false);
    1091             :         } else
    1092          36 :           LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
    1093             :       }
    1094             : 
    1095             :       // If NoGap is true, pop consecutive registers and then leave the rest
    1096             :       // for other instructions. e.g.
    1097             :       // vpop {d8, d10, d11} -> vpop {d8}, vpop {d10, d11}
    1098       11176 :       if (NoGap && LastReg && LastReg != Reg-1)
    1099             :         break;
    1100             : 
    1101       11151 :       LastReg = Reg;
    1102       11151 :       Regs.push_back(Reg);
    1103             :     }
    1104             : 
    1105       10735 :     if (Regs.empty())
    1106             :       continue;
    1107             : 
    1108             :     llvm::sort(Regs.begin(), Regs.end(), [&](unsigned LHS, unsigned RHS) {
    1109        7187 :       return TRI.getEncodingValue(LHS) < TRI.getEncodingValue(RHS);
    1110             :     });
    1111             : 
    1112        7928 :     if (Regs.size() > 1 || LdrOpc == 0) {
    1113        7376 :       MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdmOpc), ARM::SP)
    1114        3688 :                                     .addReg(ARM::SP)
    1115        3688 :                                     .add(predOps(ARMCC::AL));
    1116       14563 :       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
    1117       21750 :         MIB.addReg(Regs[i], getDefRegState(true));
    1118        3688 :       if (DeleteRet) {
    1119        2429 :         if (MI != MBB.end()) {
    1120             :           MIB.copyImplicitOps(*MI);
    1121        2429 :           MI->eraseFromParent();
    1122             :         }
    1123             :       }
    1124        3688 :       MI = MIB;
    1125         276 :     } else if (Regs.size() == 1) {
    1126             :       // If we adjusted the reg to PC from LR above, switch it back here. We
    1127             :       // only do that for LDM.
    1128         276 :       if (Regs[0] == ARM::PC)
    1129         142 :         Regs[0] = ARM::LR;
    1130             :       MachineInstrBuilder MIB =
    1131         552 :         BuildMI(MBB, MI, DL, TII.get(LdrOpc), Regs[0])
    1132         276 :           .addReg(ARM::SP, RegState::Define)
    1133         276 :           .addReg(ARM::SP);
    1134             :       // ARM mode needs an extra reg0 here due to addrmode2. Will go away once
    1135             :       // that refactoring is complete (eventually).
    1136         276 :       if (LdrOpc == ARM::LDR_POST_REG || LdrOpc == ARM::LDR_POST_IMM) {
    1137         197 :         MIB.addReg(0);
    1138             :         MIB.addImm(ARM_AM::getAM2Opc(ARM_AM::add, 4, ARM_AM::no_shift));
    1139             :       } else
    1140             :         MIB.addImm(4);
    1141         276 :       MIB.add(predOps(ARMCC::AL));
    1142             :     }
    1143             :     Regs.clear();
    1144             : 
    1145             :     // Put any subsequent vpop instructions after this one: they will refer to
    1146             :     // higher register numbers so need to be popped afterwards.
    1147        3964 :     if (MI != MBB.end())
    1148             :       ++MI;
    1149             :   }
    1150       10710 : }
    1151             : 
    1152             : /// Emit aligned spill instructions for NumAlignedDPRCS2Regs D-registers
    1153             : /// starting from d8.  Also insert stack realignment code and leave the stack
    1154             : /// pointer pointing to the d8 spill slot.
    1155          40 : static void emitAlignedDPRCS2Spills(MachineBasicBlock &MBB,
    1156             :                                     MachineBasicBlock::iterator MI,
    1157             :                                     unsigned NumAlignedDPRCS2Regs,
    1158             :                                     const std::vector<CalleeSavedInfo> &CSI,
    1159             :                                     const TargetRegisterInfo *TRI) {
    1160          40 :   MachineFunction &MF = *MBB.getParent();
    1161          40 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1162          40 :   DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
    1163          40 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1164          40 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1165             : 
    1166             :   // Mark the D-register spill slots as properly aligned.  Since MFI computes
    1167             :   // stack slot layout backwards, this can actually mean that the d-reg stack
    1168             :   // slot offsets can be wrong. The offset for d8 will always be correct.
    1169         604 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
    1170         524 :     unsigned DNum = CSI[i].getReg() - ARM::D8;
    1171         524 :     if (DNum > NumAlignedDPRCS2Regs - 1)
    1172             :       continue;
    1173         272 :     int FI = CSI[i].getFrameIdx();
    1174             :     // The even-numbered registers will be 16-byte aligned, the odd-numbered
    1175             :     // registers will be 8-byte aligned.
    1176         272 :     MFI.setObjectAlignment(FI, DNum % 2 ? 8 : 16);
    1177             : 
    1178             :     // The stack slot for D8 needs to be maximally aligned because this is
    1179             :     // actually the point where we align the stack pointer.  MachineFrameInfo
    1180             :     // computes all offsets relative to the incoming stack pointer which is a
    1181             :     // bit weird when realigning the stack.  Any extra padding for this
    1182             :     // over-alignment is not realized because the code inserted below adjusts
    1183             :     // the stack pointer by numregs * 8 before aligning the stack pointer.
    1184         272 :     if (DNum == 0)
    1185          40 :       MFI.setObjectAlignment(FI, MFI.getMaxAlignment());
    1186             :   }
    1187             : 
    1188             :   // Move the stack pointer to the d8 spill slot, and align it at the same
    1189             :   // time. Leave the stack slot address in the scratch register r4.
    1190             :   //
    1191             :   //   sub r4, sp, #numregs * 8
    1192             :   //   bic r4, r4, #align - 1
    1193             :   //   mov sp, r4
    1194             :   //
    1195          40 :   bool isThumb = AFI->isThumbFunction();
    1196             :   assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
    1197             :   AFI->setShouldRestoreSPFromFP(true);
    1198             : 
    1199             :   // sub r4, sp, #numregs * 8
    1200             :   // The immediate is <= 64, so it doesn't need any special encoding.
    1201          40 :   unsigned Opc = isThumb ? ARM::t2SUBri : ARM::SUBri;
    1202          80 :   BuildMI(MBB, MI, DL, TII.get(Opc), ARM::R4)
    1203          40 :       .addReg(ARM::SP)
    1204          40 :       .addImm(8 * NumAlignedDPRCS2Regs)
    1205          40 :       .add(predOps(ARMCC::AL))
    1206          40 :       .add(condCodeOp());
    1207             : 
    1208          40 :   unsigned MaxAlign = MF.getFrameInfo().getMaxAlignment();
    1209             :   // We must set parameter MustBeSingleInstruction to true, since
    1210             :   // skipAlignedDPRCS2Spills expects exactly 3 instructions to perform
    1211             :   // stack alignment.  Luckily, this can always be done since all ARM
    1212             :   // architecture versions that support Neon also support the BFC
    1213             :   // instruction.
    1214          40 :   emitAligningInstructions(MF, AFI, TII, MBB, MI, DL, ARM::R4, MaxAlign, true);
    1215             : 
    1216             :   // mov sp, r4
    1217             :   // The stack pointer must be adjusted before spilling anything, otherwise
    1218             :   // the stack slots could be clobbered by an interrupt handler.
    1219             :   // Leave r4 live, it is used below.
    1220          40 :   Opc = isThumb ? ARM::tMOVr : ARM::MOVr;
    1221          80 :   MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(Opc), ARM::SP)
    1222          40 :                                 .addReg(ARM::R4)
    1223          40 :                                 .add(predOps(ARMCC::AL));
    1224          40 :   if (!isThumb)
    1225          16 :     MIB.add(condCodeOp());
    1226             : 
    1227             :   // Now spill NumAlignedDPRCS2Regs registers starting from d8.
    1228             :   // r4 holds the stack slot address.
    1229             :   unsigned NextReg = ARM::D8;
    1230             : 
    1231             :   // 16-byte aligned vst1.64 with 4 d-regs and address writeback.
    1232             :   // The writeback is only needed when emitting two vst1.64 instructions.
    1233          40 :   if (NumAlignedDPRCS2Regs >= 6) {
    1234          32 :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1235             :                                                &ARM::QQPRRegClass);
    1236          32 :     MBB.addLiveIn(SupReg);
    1237          64 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1d64Qwb_fixed), ARM::R4)
    1238          32 :         .addReg(ARM::R4, RegState::Kill)
    1239             :         .addImm(16)
    1240          32 :         .addReg(NextReg)
    1241          32 :         .addReg(SupReg, RegState::ImplicitKill)
    1242          32 :         .add(predOps(ARMCC::AL));
    1243             :     NextReg += 4;
    1244          32 :     NumAlignedDPRCS2Regs -= 4;
    1245             :   }
    1246             : 
    1247             :   // We won't modify r4 beyond this point.  It currently points to the next
    1248             :   // register to be spilled.
    1249             :   unsigned R4BaseReg = NextReg;
    1250             : 
    1251             :   // 16-byte aligned vst1.64 with 4 d-regs, no writeback.
    1252          40 :   if (NumAlignedDPRCS2Regs >= 4) {
    1253          31 :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1254             :                                                &ARM::QQPRRegClass);
    1255          31 :     MBB.addLiveIn(SupReg);
    1256          62 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1d64Q))
    1257          31 :         .addReg(ARM::R4)
    1258             :         .addImm(16)
    1259          31 :         .addReg(NextReg)
    1260          31 :         .addReg(SupReg, RegState::ImplicitKill)
    1261          31 :         .add(predOps(ARMCC::AL));
    1262          31 :     NextReg += 4;
    1263          31 :     NumAlignedDPRCS2Regs -= 4;
    1264             :   }
    1265             : 
    1266             :   // 16-byte aligned vst1.64 with 2 d-regs.
    1267          40 :   if (NumAlignedDPRCS2Regs >= 2) {
    1268           9 :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1269             :                                                &ARM::QPRRegClass);
    1270           9 :     MBB.addLiveIn(SupReg);
    1271          18 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1q64))
    1272           9 :         .addReg(ARM::R4)
    1273             :         .addImm(16)
    1274           9 :         .addReg(SupReg)
    1275           9 :         .add(predOps(ARMCC::AL));
    1276           9 :     NextReg += 2;
    1277           9 :     NumAlignedDPRCS2Regs -= 2;
    1278             :   }
    1279             : 
    1280             :   // Finally, use a vanilla vstr.64 for the odd last register.
    1281          40 :   if (NumAlignedDPRCS2Regs) {
    1282           2 :     MBB.addLiveIn(NextReg);
    1283             :     // vstr.64 uses addrmode5 which has an offset scale of 4.
    1284           4 :     BuildMI(MBB, MI, DL, TII.get(ARM::VSTRD))
    1285           2 :         .addReg(NextReg)
    1286           2 :         .addReg(ARM::R4)
    1287           2 :         .addImm((NextReg - R4BaseReg) * 2)
    1288           2 :         .add(predOps(ARMCC::AL));
    1289             :   }
    1290             : 
    1291             :   // The last spill instruction inserted should kill the scratch register r4.
    1292          40 :   std::prev(MI)->addRegisterKilled(ARM::R4, TRI);
    1293          40 : }
    1294             : 
    1295             : /// Skip past the code inserted by emitAlignedDPRCS2Spills, and return an
    1296             : /// iterator to the following instruction.
    1297             : static MachineBasicBlock::iterator
    1298          40 : skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
    1299             :                         unsigned NumAlignedDPRCS2Regs) {
    1300             :   //   sub r4, sp, #numregs * 8
    1301             :   //   bic r4, r4, #align - 1
    1302             :   //   mov sp, r4
    1303             :   ++MI; ++MI; ++MI;
    1304             :   assert(MI->mayStore() && "Expecting spill instruction");
    1305             : 
    1306             :   // These switches all fall through.
    1307          40 :   switch(NumAlignedDPRCS2Regs) {
    1308             :   case 7:
    1309             :     ++MI;
    1310             :     assert(MI->mayStore() && "Expecting spill instruction");
    1311             :     LLVM_FALLTHROUGH;
    1312          33 :   default:
    1313             :     ++MI;
    1314             :     assert(MI->mayStore() && "Expecting spill instruction");
    1315             :     LLVM_FALLTHROUGH;
    1316          40 :   case 1:
    1317             :   case 2:
    1318             :   case 4:
    1319             :     assert(MI->killsRegister(ARM::R4) && "Missed kill flag");
    1320             :     ++MI;
    1321             :   }
    1322          40 :   return MI;
    1323             : }
    1324             : 
    1325             : /// Emit aligned reload instructions for NumAlignedDPRCS2Regs D-registers
    1326             : /// starting from d8.  These instructions are assumed to execute while the
    1327             : /// stack is still aligned, unlike the code inserted by emitPopInst.
    1328          32 : static void emitAlignedDPRCS2Restores(MachineBasicBlock &MBB,
    1329             :                                       MachineBasicBlock::iterator MI,
    1330             :                                       unsigned NumAlignedDPRCS2Regs,
    1331             :                                       const std::vector<CalleeSavedInfo> &CSI,
    1332             :                                       const TargetRegisterInfo *TRI) {
    1333          32 :   MachineFunction &MF = *MBB.getParent();
    1334          32 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1335          32 :   DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
    1336          32 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1337             : 
    1338             :   // Find the frame index assigned to d8.
    1339             :   int D8SpillFI = 0;
    1340         428 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i)
    1341         792 :     if (CSI[i].getReg() == ARM::D8) {
    1342          32 :       D8SpillFI = CSI[i].getFrameIdx();
    1343          32 :       break;
    1344             :     }
    1345             : 
    1346             :   // Materialize the address of the d8 spill slot into the scratch register r4.
    1347             :   // This can be fairly complicated if the stack frame is large, so just use
    1348             :   // the normal frame index elimination mechanism to do it.  This code runs as
    1349             :   // the initial part of the epilog where the stack and base pointers haven't
    1350             :   // been changed yet.
    1351          32 :   bool isThumb = AFI->isThumbFunction();
    1352             :   assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
    1353             : 
    1354          32 :   unsigned Opc = isThumb ? ARM::t2ADDri : ARM::ADDri;
    1355          64 :   BuildMI(MBB, MI, DL, TII.get(Opc), ARM::R4)
    1356             :       .addFrameIndex(D8SpillFI)
    1357             :       .addImm(0)
    1358          32 :       .add(predOps(ARMCC::AL))
    1359          32 :       .add(condCodeOp());
    1360             : 
    1361             :   // Now restore NumAlignedDPRCS2Regs registers starting from d8.
    1362             :   unsigned NextReg = ARM::D8;
    1363             : 
    1364             :   // 16-byte aligned vld1.64 with 4 d-regs and writeback.
    1365          32 :   if (NumAlignedDPRCS2Regs >= 6) {
    1366          24 :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1367             :                                                &ARM::QQPRRegClass);
    1368          48 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1d64Qwb_fixed), NextReg)
    1369          24 :         .addReg(ARM::R4, RegState::Define)
    1370          24 :         .addReg(ARM::R4, RegState::Kill)
    1371             :         .addImm(16)
    1372          24 :         .addReg(SupReg, RegState::ImplicitDefine)
    1373          24 :         .add(predOps(ARMCC::AL));
    1374             :     NextReg += 4;
    1375          24 :     NumAlignedDPRCS2Regs -= 4;
    1376             :   }
    1377             : 
    1378             :   // We won't modify r4 beyond this point.  It currently points to the next
    1379             :   // register to be spilled.
    1380             :   unsigned R4BaseReg = NextReg;
    1381             : 
    1382             :   // 16-byte aligned vld1.64 with 4 d-regs, no writeback.
    1383          32 :   if (NumAlignedDPRCS2Regs >= 4) {
    1384          23 :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1385             :                                                &ARM::QQPRRegClass);
    1386          46 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1d64Q), NextReg)
    1387          23 :         .addReg(ARM::R4)
    1388             :         .addImm(16)
    1389          23 :         .addReg(SupReg, RegState::ImplicitDefine)
    1390          23 :         .add(predOps(ARMCC::AL));
    1391          23 :     NextReg += 4;
    1392          23 :     NumAlignedDPRCS2Regs -= 4;
    1393             :   }
    1394             : 
    1395             :   // 16-byte aligned vld1.64 with 2 d-regs.
    1396          32 :   if (NumAlignedDPRCS2Regs >= 2) {
    1397           9 :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1398             :                                                &ARM::QPRRegClass);
    1399          18 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1q64), SupReg)
    1400           9 :         .addReg(ARM::R4)
    1401             :         .addImm(16)
    1402           9 :         .add(predOps(ARMCC::AL));
    1403           9 :     NextReg += 2;
    1404           9 :     NumAlignedDPRCS2Regs -= 2;
    1405             :   }
    1406             : 
    1407             :   // Finally, use a vanilla vldr.64 for the remaining odd register.
    1408          32 :   if (NumAlignedDPRCS2Regs)
    1409           4 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLDRD), NextReg)
    1410           2 :         .addReg(ARM::R4)
    1411           2 :         .addImm(2 * (NextReg - R4BaseReg))
    1412           2 :         .add(predOps(ARMCC::AL));
    1413             : 
    1414             :   // Last store kills r4.
    1415          32 :   std::prev(MI)->addRegisterKilled(ARM::R4, TRI);
    1416          32 : }
    1417             : 
    1418        3623 : bool ARMFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
    1419             :                                         MachineBasicBlock::iterator MI,
    1420             :                                         const std::vector<CalleeSavedInfo> &CSI,
    1421             :                                         const TargetRegisterInfo *TRI) const {
    1422        3623 :   if (CSI.empty())
    1423             :     return false;
    1424             : 
    1425        3623 :   MachineFunction &MF = *MBB.getParent();
    1426        3623 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1427             : 
    1428        3623 :   unsigned PushOpc = AFI->isThumbFunction() ? ARM::t2STMDB_UPD : ARM::STMDB_UPD;
    1429        3623 :   unsigned PushOneOpc = AFI->isThumbFunction() ?
    1430             :     ARM::t2STR_PRE : ARM::STR_PRE_IMM;
    1431             :   unsigned FltOpc = ARM::VSTMDDB_UPD;
    1432        3623 :   unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
    1433        3623 :   emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isARMArea1Register, 0,
    1434             :                MachineInstr::FrameSetup);
    1435        3623 :   emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isARMArea2Register, 0,
    1436             :                MachineInstr::FrameSetup);
    1437        3623 :   emitPushInst(MBB, MI, CSI, FltOpc, 0, true, &isARMArea3Register,
    1438             :                NumAlignedDPRCS2Regs, MachineInstr::FrameSetup);
    1439             : 
    1440             :   // The code above does not insert spill code for the aligned DPRCS2 registers.
    1441             :   // The stack realignment code will be inserted between the push instructions
    1442             :   // and these spills.
    1443        3623 :   if (NumAlignedDPRCS2Regs)
    1444          40 :     emitAlignedDPRCS2Spills(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
    1445             : 
    1446             :   return true;
    1447             : }
    1448             : 
    1449        3570 : bool ARMFrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
    1450             :                                         MachineBasicBlock::iterator MI,
    1451             :                                         std::vector<CalleeSavedInfo> &CSI,
    1452             :                                         const TargetRegisterInfo *TRI) const {
    1453        3570 :   if (CSI.empty())
    1454             :     return false;
    1455             : 
    1456        3570 :   MachineFunction &MF = *MBB.getParent();
    1457        3570 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1458        3570 :   bool isVarArg = AFI->getArgRegsSaveSize() > 0;
    1459        3570 :   unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
    1460             : 
    1461             :   // The emitPopInst calls below do not insert reloads for the aligned DPRCS2
    1462             :   // registers. Do that here instead.
    1463        3570 :   if (NumAlignedDPRCS2Regs)
    1464          32 :     emitAlignedDPRCS2Restores(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
    1465             : 
    1466        3570 :   unsigned PopOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
    1467        3570 :   unsigned LdrOpc = AFI->isThumbFunction() ? ARM::t2LDR_POST :ARM::LDR_POST_IMM;
    1468             :   unsigned FltOpc = ARM::VLDMDIA_UPD;
    1469        3570 :   emitPopInst(MBB, MI, CSI, FltOpc, 0, isVarArg, true, &isARMArea3Register,
    1470             :               NumAlignedDPRCS2Regs);
    1471        3570 :   emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false,
    1472             :               &isARMArea2Register, 0);
    1473        3570 :   emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false,
    1474             :               &isARMArea1Register, 0);
    1475             : 
    1476        3570 :   return true;
    1477             : }
    1478             : 
    1479             : // FIXME: Make generic?
    1480         644 : static unsigned GetFunctionSizeInBytes(const MachineFunction &MF,
    1481             :                                        const ARMBaseInstrInfo &TII) {
    1482             :   unsigned FnSize = 0;
    1483        1964 :   for (auto &MBB : MF) {
    1484        7061 :     for (auto &MI : MBB)
    1485        5741 :       FnSize += TII.getInstSizeInBytes(MI);
    1486             :   }
    1487         644 :   return FnSize;
    1488             : }
    1489             : 
    1490             : /// estimateRSStackSizeLimit - Look at each instruction that references stack
    1491             : /// frames and return the stack size limit beyond which some of these
    1492             : /// instructions will require a scratch register during their expansion later.
    1493             : // FIXME: Move to TII?
    1494       14529 : static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
    1495             :                                          const TargetFrameLowering *TFI) {
    1496       14529 :   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1497       14529 :   unsigned Limit = (1 << 12) - 1;
    1498       34913 :   for (auto &MBB : MF) {
    1499      141242 :     for (auto &MI : MBB) {
    1500      627882 :       for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    1501     1042456 :         if (!MI.getOperand(i).isFI())
    1502             :           continue;
    1503             : 
    1504             :         // When using ADDri to get the address of a stack object, 255 is the
    1505             :         // largest offset guaranteed to fit in the immediate offset.
    1506       28408 :         if (MI.getOpcode() == ARM::ADDri) {
    1507        1254 :           Limit = std::min(Limit, (1U << 8) - 1);
    1508        1254 :           break;
    1509             :         }
    1510             : 
    1511             :         // Otherwise check the addressing mode.
    1512       12950 :         switch (MI.getDesc().TSFlags & ARMII::AddrModeMask) {
    1513          70 :         case ARMII::AddrMode3:
    1514             :         case ARMII::AddrModeT2_i8:
    1515          70 :           Limit = std::min(Limit, (1U << 8) - 1);
    1516          70 :           break;
    1517        1232 :         case ARMII::AddrMode5:
    1518             :         case ARMII::AddrModeT2_i8s4:
    1519             :         case ARMII::AddrModeT2_ldrex:
    1520        1232 :           Limit = std::min(Limit, ((1U << 8) - 1) * 4);
    1521        1232 :           break;
    1522        3080 :         case ARMII::AddrModeT2_i12:
    1523             :           // i12 supports only positive offset so these will be converted to
    1524             :           // i8 opcodes. See llvm::rewriteT2FrameIndex.
    1525        3080 :           if (TFI->hasFP(MF) && AFI->hasStackFrame())
    1526           0 :             Limit = std::min(Limit, (1U << 8) - 1);
    1527             :           break;
    1528             :         case ARMII::AddrMode4:
    1529             :         case ARMII::AddrMode6:
    1530             :           // Addressing modes 4 & 6 (load/store) instructions can't encode an
    1531             :           // immediate offset for stack references.
    1532             :           return 0;
    1533             :         default:
    1534             :           break;
    1535             :         }
    1536             :         break; // At most one FI per instruction
    1537             :       }
    1538             :     }
    1539             :   }
    1540             : 
    1541       14508 :   return Limit;
    1542             : }
    1543             : 
    1544             : // In functions that realign the stack, it can be an advantage to spill the
    1545             : // callee-saved vector registers after realigning the stack. The vst1 and vld1
    1546             : // instructions take alignment hints that can improve performance.
    1547             : static void
    1548       14529 : checkNumAlignedDPRCS2Regs(MachineFunction &MF, BitVector &SavedRegs) {
    1549       14529 :   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(0);
    1550       14529 :   if (!SpillAlignedNEONRegs)
    1551             :     return;
    1552             : 
    1553             :   // Naked functions don't spill callee-saved registers.
    1554       14526 :   if (MF.getFunction().hasFnAttribute(Attribute::Naked))
    1555             :     return;
    1556             : 
    1557             :   // We are planning to use NEON instructions vst1 / vld1.
    1558       14525 :   if (!static_cast<const ARMSubtarget &>(MF.getSubtarget()).hasNEON())
    1559             :     return;
    1560             : 
    1561             :   // Don't bother if the default stack alignment is sufficiently high.
    1562        8345 :   if (MF.getSubtarget().getFrameLowering()->getStackAlignment() >= 8)
    1563             :     return;
    1564             : 
    1565             :   // Aligned spills require stack realignment.
    1566        1839 :   if (!static_cast<const ARMBaseRegisterInfo *>(
    1567        1839 :            MF.getSubtarget().getRegisterInfo())->canRealignStack(MF))
    1568             :     return;
    1569             : 
    1570             :   // We always spill contiguous d-registers starting from d8. Count how many
    1571             :   // needs spilling.  The register allocator will almost always use the
    1572             :   // callee-saved registers in order, but it can happen that there are holes in
    1573             :   // the range.  Registers above the hole will be spilled to the standard DPRCS
    1574             :   // area.
    1575             :   unsigned NumSpills = 0;
    1576        1106 :   for (; NumSpills < 8; ++NumSpills)
    1577        2150 :     if (!SavedRegs.test(ARM::D8 + NumSpills))
    1578             :       break;
    1579             : 
    1580             :   // Don't do this for just one d-register. It's not worth it.
    1581         829 :   if (NumSpills < 2)
    1582             :     return;
    1583             : 
    1584             :   // Spill the first NumSpills D-registers after realigning the stack.
    1585          40 :   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(NumSpills);
    1586             : 
    1587             :   // A scratch register is required for the vst1 / vld1 instructions.
    1588             :   SavedRegs.set(ARM::R4);
    1589             : }
    1590             : 
    1591       14529 : void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
    1592             :                                             BitVector &SavedRegs,
    1593             :                                             RegScavenger *RS) const {
    1594       14529 :   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
    1595             :   // This tells PEI to spill the FP as if it is any other callee-save register
    1596             :   // to take advantage the eliminateFrameIndex machinery. This also ensures it
    1597             :   // is spilled in the order specified by getCalleeSavedRegs() to make it easier
    1598             :   // to combine multiple loads / stores.
    1599             :   bool CanEliminateFrame = true;
    1600             :   bool CS1Spilled = false;
    1601             :   bool LRSpilled = false;
    1602             :   unsigned NumGPRSpills = 0;
    1603             :   unsigned NumFPRSpills = 0;
    1604             :   SmallVector<unsigned, 4> UnspilledCS1GPRs;
    1605             :   SmallVector<unsigned, 4> UnspilledCS2GPRs;
    1606             :   const ARMBaseRegisterInfo *RegInfo = static_cast<const ARMBaseRegisterInfo *>(
    1607       14529 :       MF.getSubtarget().getRegisterInfo());
    1608             :   const ARMBaseInstrInfo &TII =
    1609       14529 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    1610       14529 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1611       14529 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1612       14529 :   MachineRegisterInfo &MRI = MF.getRegInfo();
    1613       14529 :   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1614             :   (void)TRI;  // Silence unused warning in non-assert builds.
    1615       14529 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
    1616             : 
    1617             :   // Spill R4 if Thumb2 function requires stack realignment - it will be used as
    1618             :   // scratch register. Also spill R4 if Thumb2 function has varsized objects,
    1619             :   // since it's not always possible to restore sp from fp in a single
    1620             :   // instruction.
    1621             :   // FIXME: It will be better just to find spare register here.
    1622       14529 :   if (AFI->isThumb2Function() &&
    1623        4891 :       (MFI.hasVarSizedObjects() || RegInfo->needsStackRealignment(MF)))
    1624             :     SavedRegs.set(ARM::R4);
    1625             : 
    1626             :   // If a stack probe will be emitted, spill R4 and LR, since they are
    1627             :   // clobbered by the stack probe call.
    1628             :   // This estimate should be a safe, conservative estimate. The actual
    1629             :   // stack probe is enabled based on the size of the local objects;
    1630             :   // this estimate also includes the varargs store size.
    1631       29230 :   if (STI.isTargetWindows() &&
    1632         172 :       WindowsRequiresStackProbe(MF, MFI.estimateStackSize(MF))) {
    1633             :     SavedRegs.set(ARM::R4);
    1634             :     SavedRegs.set(ARM::LR);
    1635             :   }
    1636             : 
    1637       14529 :   if (AFI->isThumb1OnlyFunction()) {
    1638             :     // Spill LR if Thumb1 function uses variable length argument lists.
    1639        1160 :     if (AFI->getArgRegsSaveSize() > 0)
    1640             :       SavedRegs.set(ARM::LR);
    1641             : 
    1642             :     // Spill R4 if Thumb1 epilogue has to restore SP from FP or the function
    1643             :     // requires stack alignment.  We don't know for sure what the stack size
    1644             :     // will be, but for this, an estimate is good enough. If there anything
    1645             :     // changes it, it'll be a spill, which implies we've used all the registers
    1646             :     // and so R4 is already used, so not marking it here will be OK.
    1647             :     // FIXME: It will be better just to find spare register here.
    1648        2267 :     if (MFI.hasVarSizedObjects() || RegInfo->needsStackRealignment(MF) ||
    1649        1107 :         MFI.estimateStackSize(MF) > 508)
    1650             :       SavedRegs.set(ARM::R4);
    1651             :   }
    1652             : 
    1653             :   // See if we can spill vector registers to aligned stack.
    1654       14529 :   checkNumAlignedDPRCS2Regs(MF, SavedRegs);
    1655             : 
    1656             :   // Spill the BasePtr if it's used.
    1657       14529 :   if (RegInfo->hasBasePointer(MF))
    1658          24 :     SavedRegs.set(RegInfo->getBaseRegister());
    1659             : 
    1660             :   // Don't spill FP if the frame can be eliminated. This is determined
    1661             :   // by scanning the callee-save registers to see if any is modified.
    1662       14529 :   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
    1663      259374 :   for (unsigned i = 0; CSRegs[i]; ++i) {
    1664      244845 :     unsigned Reg = CSRegs[i];
    1665             :     bool Spilled = false;
    1666      244845 :     if (SavedRegs.test(Reg)) {
    1667             :       Spilled = true;
    1668             :       CanEliminateFrame = false;
    1669             :     }
    1670             : 
    1671      244845 :     if (!ARM::GPRRegClass.contains(Reg)) {
    1672      116480 :       if (Spilled) {
    1673        1335 :         if (ARM::SPRRegClass.contains(Reg))
    1674           0 :           NumFPRSpills++;
    1675        1335 :         else if (ARM::DPRRegClass.contains(Reg))
    1676        1335 :           NumFPRSpills += 2;
    1677           0 :         else if (ARM::QPRRegClass.contains(Reg))
    1678           0 :           NumFPRSpills += 4;
    1679             :       }
    1680      225907 :       continue;
    1681             :     }
    1682             : 
    1683      128365 :     if (Spilled) {
    1684        8636 :       NumGPRSpills++;
    1685             : 
    1686        8636 :       if (!STI.splitFramePushPop(MF)) {
    1687        6279 :         if (Reg == ARM::LR)
    1688             :           LRSpilled = true;
    1689             :         CS1Spilled = true;
    1690        6279 :         continue;
    1691             :       }
    1692             : 
    1693             :       // Keep track if LR and any of R4, R5, R6, and R7 is spilled.
    1694        2357 :       switch (Reg) {
    1695         998 :       case ARM::LR:
    1696             :         LRSpilled = true;
    1697             :         LLVM_FALLTHROUGH;
    1698             :       case ARM::R0: case ARM::R1:
    1699             :       case ARM::R2: case ARM::R3:
    1700             :       case ARM::R4: case ARM::R5:
    1701             :       case ARM::R6: case ARM::R7:
    1702             :         CS1Spilled = true;
    1703             :         break;
    1704             :       default:
    1705             :         break;
    1706             :       }
    1707             :     } else {
    1708      119729 :       if (!STI.splitFramePushPop(MF)) {
    1709      103148 :         UnspilledCS1GPRs.push_back(Reg);
    1710      103148 :         continue;
    1711             :       }
    1712             : 
    1713       16581 :       switch (Reg) {
    1714        9032 :       case ARM::R0: case ARM::R1:
    1715             :       case ARM::R2: case ARM::R3:
    1716             :       case ARM::R4: case ARM::R5:
    1717             :       case ARM::R6: case ARM::R7:
    1718             :       case ARM::LR:
    1719        9032 :         UnspilledCS1GPRs.push_back(Reg);
    1720        9032 :         break;
    1721        7549 :       default:
    1722        7549 :         UnspilledCS2GPRs.push_back(Reg);
    1723        7549 :         break;
    1724             :       }
    1725             :     }
    1726             :   }
    1727             : 
    1728             :   bool ForceLRSpill = false;
    1729       14529 :   if (!LRSpilled && AFI->isThumb1OnlyFunction()) {
    1730         644 :     unsigned FnSize = GetFunctionSizeInBytes(MF, TII);
    1731             :     // Force LR to be spilled if the Thumb function size is > 2048. This enables
    1732             :     // use of BL to implement far jump. If it turns out that it's not needed
    1733             :     // then the branch fix up path will undo it.
    1734         644 :     if (FnSize >= (1 << 11)) {
    1735             :       CanEliminateFrame = false;
    1736             :       ForceLRSpill = true;
    1737             :     }
    1738             :   }
    1739             : 
    1740             :   // If any of the stack slot references may be out of range of an immediate
    1741             :   // offset, make sure a register (or a spill slot) is available for the
    1742             :   // register scavenger. Note that if we're indexing off the frame pointer, the
    1743             :   // effective stack size is 4 bytes larger since the FP points to the stack
    1744             :   // slot of the previous FP. Also, if we have variable sized objects in the
    1745             :   // function, stack slot references will often be negative, and some of
    1746             :   // our instructions are positive-offset only, so conservatively consider
    1747             :   // that case to want a spill slot (or register) as well. Similarly, if
    1748             :   // the function adjusts the stack pointer during execution and the
    1749             :   // adjustments aren't already part of our stack size estimate, our offset
    1750             :   // calculations may be off, so be conservative.
    1751             :   // FIXME: We could add logic to be more precise about negative offsets
    1752             :   //        and which instructions will need a scratch register for them. Is it
    1753             :   //        worth the effort and added fragility?
    1754             :   unsigned EstimatedStackSize =
    1755       14529 :       MFI.estimateStackSize(MF) + 4 * (NumGPRSpills + NumFPRSpills);
    1756             : 
    1757             :   // Determine biggest (positive) SP offset in MachineFrameInfo.
    1758       14529 :   int MaxFixedOffset = 0;
    1759       16140 :   for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
    1760        3222 :     int MaxObjectOffset = MFI.getObjectOffset(I) + MFI.getObjectSize(I);
    1761        1611 :     MaxFixedOffset = std::max(MaxFixedOffset, MaxObjectOffset);
    1762             :   }
    1763             : 
    1764       14529 :   bool HasFP = hasFP(MF);
    1765       14529 :   if (HasFP) {
    1766        1927 :     if (AFI->hasStackFrame())
    1767          15 :       EstimatedStackSize += 4;
    1768             :   } else {
    1769             :     // If FP is not used, SP will be used to access arguments, so count the
    1770             :     // size of arguments into the estimation.
    1771       12602 :     EstimatedStackSize += MaxFixedOffset;
    1772             :   }
    1773       14529 :   EstimatedStackSize += 16; // For possible paddings.
    1774             : 
    1775       14529 :   unsigned EstimatedRSStackSizeLimit = estimateRSStackSizeLimit(MF, this);
    1776             :   int MaxFPOffset = getMaxFPOffset(MF.getFunction(), *AFI);
    1777       14362 :   bool BigFrameOffsets = EstimatedStackSize >= EstimatedRSStackSizeLimit ||
    1778       14362 :     MFI.hasVarSizedObjects() ||
    1779       28841 :     (MFI.adjustsStack() && !canSimplifyCallFramePseudos(MF)) ||
    1780             :     // For large argument stacks fp relative addressed may overflow.
    1781        1817 :     (HasFP && (MaxFixedOffset - MaxFPOffset) >= (int)EstimatedRSStackSizeLimit);
    1782       14529 :   if (BigFrameOffsets ||
    1783       14529 :       !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF)) {
    1784             :     AFI->setHasStackFrame(true);
    1785             : 
    1786        4317 :     if (HasFP) {
    1787        1137 :       SavedRegs.set(FramePtr);
    1788             :       // If the frame pointer is required by the ABI, also spill LR so that we
    1789             :       // emit a complete frame record.
    1790        1137 :       if (MF.getTarget().Options.DisableFramePointerElim(MF) && !LRSpilled) {
    1791             :         SavedRegs.set(ARM::LR);
    1792             :         LRSpilled = true;
    1793         118 :         NumGPRSpills++;
    1794         118 :         auto LRPos = llvm::find(UnspilledCS1GPRs, ARM::LR);
    1795         118 :         if (LRPos != UnspilledCS1GPRs.end())
    1796         118 :           UnspilledCS1GPRs.erase(LRPos);
    1797             :       }
    1798             :       auto FPPos = llvm::find(UnspilledCS1GPRs, FramePtr);
    1799        1137 :       if (FPPos != UnspilledCS1GPRs.end())
    1800        1063 :         UnspilledCS1GPRs.erase(FPPos);
    1801        1137 :       NumGPRSpills++;
    1802        1137 :       if (FramePtr == ARM::R7)
    1803             :         CS1Spilled = true;
    1804             :     }
    1805             : 
    1806             :     // This is true when we inserted a spill for an unused register that can now
    1807             :     // be used for register scavenging.
    1808             :     bool ExtraCSSpill = false;
    1809             : 
    1810        4317 :     if (AFI->isThumb1OnlyFunction()) {
    1811             :       // For Thumb1-only targets, we need some low registers when we save and
    1812             :       // restore the high registers (which aren't allocatable, but could be
    1813             :       // used by inline assembly) because the push/pop instructions can not
    1814             :       // access high registers. If necessary, we might need to push more low
    1815             :       // registers to ensure that there is at least one free that can be used
    1816             :       // for the saving & restoring, and preferably we should ensure that as
    1817             :       // many as are needed are available so that fewer push/pop instructions
    1818             :       // are required.
    1819             : 
    1820             :       // Low registers which are not currently pushed, but could be (r4-r7).
    1821             :       SmallVector<unsigned, 4> AvailableRegs;
    1822             : 
    1823             :       // Unused argument registers (r0-r3) can be clobbered in the prologue for
    1824             :       // free.
    1825         634 :       int EntryRegDeficit = 0;
    1826        3170 :       for (unsigned Reg : {ARM::R0, ARM::R1, ARM::R2, ARM::R3}) {
    1827        2536 :         if (!MF.getRegInfo().isLiveIn(Reg)) {
    1828        1846 :           --EntryRegDeficit;
    1829             :           LLVM_DEBUG(dbgs()
    1830             :                      << printReg(Reg, TRI)
    1831             :                      << " is unused argument register, EntryRegDeficit = "
    1832             :                      << EntryRegDeficit << "\n");
    1833             :         }
    1834             :       }
    1835             : 
    1836             :       // Unused return registers can be clobbered in the epilogue for free.
    1837         634 :       int ExitRegDeficit = AFI->getReturnRegsCount() - 4;
    1838             :       LLVM_DEBUG(dbgs() << AFI->getReturnRegsCount()
    1839             :                         << " return regs used, ExitRegDeficit = "
    1840             :                         << ExitRegDeficit << "\n");
    1841             : 
    1842         634 :       int RegDeficit = std::max(EntryRegDeficit, ExitRegDeficit);
    1843             :       LLVM_DEBUG(dbgs() << "RegDeficit = " << RegDeficit << "\n");
    1844             : 
    1845             :       // r4-r6 can be used in the prologue if they are pushed by the first push
    1846             :       // instruction.
    1847        2536 :       for (unsigned Reg : {ARM::R4, ARM::R5, ARM::R6}) {
    1848        1902 :         if (SavedRegs.test(Reg)) {
    1849         618 :           --RegDeficit;
    1850             :           LLVM_DEBUG(dbgs() << printReg(Reg, TRI)
    1851             :                             << " is saved low register, RegDeficit = "
    1852             :                             << RegDeficit << "\n");
    1853             :         } else {
    1854        1284 :           AvailableRegs.push_back(Reg);
    1855             :           LLVM_DEBUG(
    1856             :               dbgs()
    1857             :               << printReg(Reg, TRI)
    1858             :               << " is non-saved low register, adding to AvailableRegs\n");
    1859             :         }
    1860             :       }
    1861             : 
    1862             :       // r7 can be used if it is not being used as the frame pointer.
    1863         634 :       if (!HasFP) {
    1864         476 :         if (SavedRegs.test(ARM::R7)) {
    1865          41 :           --RegDeficit;
    1866             :           LLVM_DEBUG(dbgs() << "%r7 is saved low register, RegDeficit = "
    1867             :                             << RegDeficit << "\n");
    1868             :         } else {
    1869         435 :           AvailableRegs.push_back(ARM::R7);
    1870             :           LLVM_DEBUG(
    1871             :               dbgs()
    1872             :               << "%r7 is non-saved low register, adding to AvailableRegs\n");
    1873             :         }
    1874             :       }
    1875             : 
    1876             :       // Each of r8-r11 needs to be copied to a low register, then pushed.
    1877        3170 :       for (unsigned Reg : {ARM::R8, ARM::R9, ARM::R10, ARM::R11}) {
    1878        2536 :         if (SavedRegs.test(Reg)) {
    1879          54 :           ++RegDeficit;
    1880             :           LLVM_DEBUG(dbgs() << printReg(Reg, TRI)
    1881             :                             << " is saved high register, RegDeficit = "
    1882             :                             << RegDeficit << "\n");
    1883             :         }
    1884             :       }
    1885             : 
    1886             :       // LR can only be used by PUSH, not POP, and can't be used at all if the
    1887             :       // llvm.returnaddress intrinsic is used. This is only worth doing if we
    1888             :       // are more limited at function entry than exit.
    1889         862 :       if ((EntryRegDeficit > ExitRegDeficit) &&
    1890         228 :           !(MF.getRegInfo().isLiveIn(ARM::LR) &&
    1891           0 :             MF.getFrameInfo().isReturnAddressTaken())) {
    1892         228 :         if (SavedRegs.test(ARM::LR)) {
    1893         180 :           --RegDeficit;
    1894             :           LLVM_DEBUG(dbgs() << "%lr is saved register, RegDeficit = "
    1895             :                             << RegDeficit << "\n");
    1896             :         } else {
    1897          48 :           AvailableRegs.push_back(ARM::LR);
    1898             :           LLVM_DEBUG(dbgs() << "%lr is not saved, adding to AvailableRegs\n");
    1899             :         }
    1900             :       }
    1901             : 
    1902             :       // If there are more high registers that need pushing than low registers
    1903             :       // available, push some more low registers so that we can use fewer push
    1904             :       // instructions. This might not reduce RegDeficit all the way to zero,
    1905             :       // because we can only guarantee that r4-r6 are available, but r8-r11 may
    1906             :       // need saving.
    1907             :       LLVM_DEBUG(dbgs() << "Final RegDeficit = " << RegDeficit << "\n");
    1908         642 :       for (; RegDeficit > 0 && !AvailableRegs.empty(); --RegDeficit) {
    1909           8 :         unsigned Reg = AvailableRegs.pop_back_val();
    1910             :         LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
    1911             :                           << " to make up reg deficit\n");
    1912             :         SavedRegs.set(Reg);
    1913           8 :         NumGPRSpills++;
    1914             :         CS1Spilled = true;
    1915             :         assert(!MRI.isReserved(Reg) && "Should not be reserved");
    1916           8 :         if (!MRI.isPhysRegUsed(Reg))
    1917             :           ExtraCSSpill = true;
    1918           8 :         UnspilledCS1GPRs.erase(llvm::find(UnspilledCS1GPRs, Reg));
    1919           8 :         if (Reg == ARM::LR)
    1920             :           LRSpilled = true;
    1921             :       }
    1922             :       LLVM_DEBUG(dbgs() << "After adding spills, RegDeficit = " << RegDeficit
    1923             :                         << "\n");
    1924             :     }
    1925             : 
    1926             :     // Avoid spilling LR in Thumb1 if there's a tail call: it's expensive to
    1927             :     // restore LR in that case.
    1928        4951 :     bool ExpensiveLRRestore = AFI->isThumb1OnlyFunction() && MFI.hasTailCall();
    1929             : 
    1930             :     // If LR is not spilled, but at least one of R4, R5, R6, and R7 is spilled.
    1931             :     // Spill LR as well so we can fold BX_RET to the registers restore (LDM).
    1932        4317 :     if (!LRSpilled && CS1Spilled && !ExpensiveLRRestore) {
    1933             :       SavedRegs.set(ARM::LR);
    1934         195 :       NumGPRSpills++;
    1935             :       SmallVectorImpl<unsigned>::iterator LRPos;
    1936         195 :       LRPos = llvm::find(UnspilledCS1GPRs, (unsigned)ARM::LR);
    1937         195 :       if (LRPos != UnspilledCS1GPRs.end())
    1938         195 :         UnspilledCS1GPRs.erase(LRPos);
    1939             : 
    1940             :       ForceLRSpill = false;
    1941         195 :       if (!MRI.isReserved(ARM::LR) && !MRI.isPhysRegUsed(ARM::LR))
    1942             :         ExtraCSSpill = true;
    1943             :     }
    1944             : 
    1945             :     // If stack and double are 8-byte aligned and we are spilling an odd number
    1946             :     // of GPRs, spill one extra callee save GPR so we won't have to pad between
    1947             :     // the integer and double callee save areas.
    1948             :     LLVM_DEBUG(dbgs() << "NumGPRSpills = " << NumGPRSpills << "\n");
    1949        4317 :     unsigned TargetAlign = getStackAlignment();
    1950        4317 :     if (TargetAlign >= 8 && (NumGPRSpills & 1)) {
    1951        2110 :       if (CS1Spilled && !UnspilledCS1GPRs.empty()) {
    1952        3748 :         for (unsigned i = 0, e = UnspilledCS1GPRs.size(); i != e; ++i) {
    1953        3739 :           unsigned Reg = UnspilledCS1GPRs[i];
    1954             :           // Don't spill high register if the function is thumb.  In the case of
    1955             :           // Windows on ARM, accept R11 (frame pointer)
    1956        2530 :           if (!AFI->isThumbFunction() ||
    1957        2530 :               (STI.isTargetWindows() && Reg == ARM::R11) ||
    1958        3739 :               isARMLowRegister(Reg) ||
    1959        1749 :               (Reg == ARM::LR && !ExpensiveLRRestore)) {
    1960             :             SavedRegs.set(Reg);
    1961             :             LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
    1962             :                               << " to make up alignment\n");
    1963        1990 :             if (!MRI.isReserved(Reg) && !MRI.isPhysRegUsed(Reg))
    1964             :               ExtraCSSpill = true;
    1965             :             break;
    1966             :           }
    1967             :         }
    1968         111 :       } else if (!UnspilledCS2GPRs.empty() && !AFI->isThumb1OnlyFunction()) {
    1969           8 :         unsigned Reg = UnspilledCS2GPRs.front();
    1970             :         SavedRegs.set(Reg);
    1971             :         LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
    1972             :                           << " to make up alignment\n");
    1973           8 :         if (!MRI.isReserved(Reg) && !MRI.isPhysRegUsed(Reg))
    1974             :           ExtraCSSpill = true;
    1975             :       }
    1976             :     }
    1977             : 
    1978             :     // Estimate if we might need to scavenge a register at some point in order
    1979             :     // to materialize a stack offset. If so, either spill one additional
    1980             :     // callee-saved register or reserve a special spill slot to facilitate
    1981             :     // register scavenging. Thumb1 needs a spill slot for stack pointer
    1982             :     // adjustments also, even when the frame itself is small.
    1983        4317 :     if (BigFrameOffsets && !ExtraCSSpill) {
    1984             :       // If any non-reserved CS register isn't spilled, just spill one or two
    1985             :       // extra. That should take care of it!
    1986         131 :       unsigned NumExtras = TargetAlign / 4;
    1987             :       SmallVector<unsigned, 2> Extras;
    1988         292 :       while (NumExtras && !UnspilledCS1GPRs.empty()) {
    1989         161 :         unsigned Reg = UnspilledCS1GPRs.back();
    1990             :         UnspilledCS1GPRs.pop_back();
    1991         161 :         if (!MRI.isReserved(Reg) &&
    1992         161 :             (!AFI->isThumb1OnlyFunction() || isARMLowRegister(Reg) ||
    1993             :              Reg == ARM::LR)) {
    1994         161 :           Extras.push_back(Reg);
    1995         161 :           NumExtras--;
    1996             :         }
    1997             :       }
    1998             :       // For non-Thumb1 functions, also check for hi-reg CS registers
    1999         131 :       if (!AFI->isThumb1OnlyFunction()) {
    2000         104 :         while (NumExtras && !UnspilledCS2GPRs.empty()) {
    2001           2 :           unsigned Reg = UnspilledCS2GPRs.back();
    2002             :           UnspilledCS2GPRs.pop_back();
    2003           2 :           if (!MRI.isReserved(Reg)) {
    2004           2 :             Extras.push_back(Reg);
    2005           2 :             NumExtras--;
    2006             :           }
    2007             :         }
    2008             :       }
    2009         131 :       if (NumExtras == 0) {
    2010         252 :         for (unsigned Reg : Extras) {
    2011             :           SavedRegs.set(Reg);
    2012         153 :           if (!MRI.isPhysRegUsed(Reg))
    2013             :             ExtraCSSpill = true;
    2014             :         }
    2015             :       }
    2016         131 :       if (!ExtraCSSpill && !AFI->isThumb1OnlyFunction()) {
    2017             :         // note: Thumb1 functions spill to R12, not the stack.  Reserve a slot
    2018             :         // closest to SP or frame pointer.
    2019             :         assert(RS && "Register scavenging not provided");
    2020             :         const TargetRegisterClass &RC = ARM::GPRRegClass;
    2021             :         unsigned Size = TRI->getSpillSize(RC);
    2022             :         unsigned Align = TRI->getSpillAlignment(RC);
    2023          17 :         RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
    2024             :       }
    2025             :     }
    2026             :   }
    2027             : 
    2028       14529 :   if (ForceLRSpill) {
    2029             :     SavedRegs.set(ARM::LR);
    2030             :     AFI->setLRIsSpilledForFarJump(true);
    2031             :   }
    2032       14529 : }
    2033             : 
    2034       13364 : MachineBasicBlock::iterator ARMFrameLowering::eliminateCallFramePseudoInstr(
    2035             :     MachineFunction &MF, MachineBasicBlock &MBB,
    2036             :     MachineBasicBlock::iterator I) const {
    2037             :   const ARMBaseInstrInfo &TII =
    2038       13364 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    2039       13364 :   if (!hasReservedCallFrame(MF)) {
    2040             :     // If we have alloca, convert as follows:
    2041             :     // ADJCALLSTACKDOWN -> sub, sp, sp, amount
    2042             :     // ADJCALLSTACKUP   -> add, sp, sp, amount
    2043             :     MachineInstr &Old = *I;
    2044             :     DebugLoc dl = Old.getDebugLoc();
    2045         170 :     unsigned Amount = TII.getFrameSize(Old);
    2046         170 :     if (Amount != 0) {
    2047             :       // We need to keep the stack aligned properly.  To do this, we round the
    2048             :       // amount of space needed for the outgoing arguments up to the next
    2049             :       // alignment boundary.
    2050          42 :       Amount = alignSPAdjust(Amount);
    2051             : 
    2052          42 :       ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    2053             :       assert(!AFI->isThumb1OnlyFunction() &&
    2054             :              "This eliminateCallFramePseudoInstr does not support Thumb1!");
    2055          42 :       bool isARM = !AFI->isThumbFunction();
    2056             : 
    2057             :       // Replace the pseudo instruction with a new instruction...
    2058          42 :       unsigned Opc = Old.getOpcode();
    2059          42 :       int PIdx = Old.findFirstPredOperandIdx();
    2060             :       ARMCC::CondCodes Pred =
    2061          42 :           (PIdx == -1) ? ARMCC::AL
    2062          84 :                        : (ARMCC::CondCodes)Old.getOperand(PIdx).getImm();
    2063             :       unsigned PredReg = TII.getFramePred(Old);
    2064          42 :       if (Opc == ARM::ADJCALLSTACKDOWN || Opc == ARM::tADJCALLSTACKDOWN) {
    2065          21 :         emitSPUpdate(isARM, MBB, I, dl, TII, -Amount, MachineInstr::NoFlags,
    2066             :                      Pred, PredReg);
    2067             :       } else {
    2068             :         assert(Opc == ARM::ADJCALLSTACKUP || Opc == ARM::tADJCALLSTACKUP);
    2069          21 :         emitSPUpdate(isARM, MBB, I, dl, TII, Amount, MachineInstr::NoFlags,
    2070             :                      Pred, PredReg);
    2071             :       }
    2072             :     }
    2073             :   }
    2074       13364 :   return MBB.erase(I);
    2075             : }
    2076             : 
    2077             : /// Get the minimum constant for ARM that is greater than or equal to the
    2078             : /// argument. In ARM, constants can have any value that can be produced by
    2079             : /// rotating an 8-bit value to the right by an even number of bits within a
    2080             : /// 32-bit word.
    2081         112 : static uint32_t alignToARMConstant(uint32_t Value) {
    2082             :   unsigned Shifted = 0;
    2083             : 
    2084         112 :   if (Value == 0)
    2085             :       return 0;
    2086             : 
    2087         648 :   while (!(Value & 0xC0000000)) {
    2088         596 :       Value = Value << 2;
    2089         596 :       Shifted += 2;
    2090             :   }
    2091             : 
    2092          52 :   bool Carry = (Value & 0x00FFFFFF);
    2093          52 :   Value = ((Value & 0xFF000000) >> 24) + Carry;
    2094             : 
    2095          52 :   if (Value & 0x0000100)
    2096             :       Value = Value & 0x000001FC;
    2097             : 
    2098          52 :   if (Shifted > 24)
    2099          36 :       Value = Value >> (Shifted - 24);
    2100             :   else
    2101          16 :       Value = Value << (24 - Shifted);
    2102             : 
    2103             :   return Value;
    2104             : }
    2105             : 
    2106             : // The stack limit in the TCB is set to this many bytes above the actual
    2107             : // stack limit.
    2108             : static const uint64_t kSplitStackAvailable = 256;
    2109             : 
    2110             : // Adjust the function prologue to enable split stacks. This currently only
    2111             : // supports android and linux.
    2112             : //
    2113             : // The ABI of the segmented stack prologue is a little arbitrarily chosen, but
    2114             : // must be well defined in order to allow for consistent implementations of the
    2115             : // __morestack helper function. The ABI is also not a normal ABI in that it
    2116             : // doesn't follow the normal calling conventions because this allows the
    2117             : // prologue of each function to be optimized further.
    2118             : //
    2119             : // Currently, the ABI looks like (when calling __morestack)
    2120             : //
    2121             : //  * r4 holds the minimum stack size requested for this function call
    2122             : //  * r5 holds the stack size of the arguments to the function
    2123             : //  * the beginning of the function is 3 instructions after the call to
    2124             : //    __morestack
    2125             : //
    2126             : // Implementations of __morestack should use r4 to allocate a new stack, r5 to
    2127             : // place the arguments on to the new stack, and the 3-instruction knowledge to
    2128             : // jump directly to the body of the function when working on the new stack.
    2129             : //
    2130             : // An old (and possibly no longer compatible) implementation of __morestack for
    2131             : // ARM can be found at [1].
    2132             : //
    2133             : // [1] - https://github.com/mozilla/rust/blob/86efd9/src/rt/arch/arm/morestack.S
    2134          64 : void ARMFrameLowering::adjustForSegmentedStacks(
    2135             :     MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
    2136             :   unsigned Opcode;
    2137             :   unsigned CFIIndex;
    2138          64 :   const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
    2139          64 :   bool Thumb = ST->isThumb();
    2140             : 
    2141             :   // Sadly, this currently doesn't support varargs, platforms other than
    2142             :   // android/linux. Note that thumb1/thumb2 are support for android/linux.
    2143         128 :   if (MF.getFunction().isVarArg())
    2144           0 :     report_fatal_error("Segmented stacks do not support vararg functions.");
    2145          64 :   if (!ST->isTargetAndroid() && !ST->isTargetLinux())
    2146           0 :     report_fatal_error("Segmented stacks not supported on this platform.");
    2147             : 
    2148          64 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    2149          64 :   MachineModuleInfo &MMI = MF.getMMI();
    2150             :   MCContext &Context = MMI.getContext();
    2151          64 :   const MCRegisterInfo *MRI = Context.getRegisterInfo();
    2152             :   const ARMBaseInstrInfo &TII =
    2153          64 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    2154          64 :   ARMFunctionInfo *ARMFI = MF.getInfo<ARMFunctionInfo>();
    2155          64 :   DebugLoc DL;
    2156             : 
    2157          64 :   uint64_t StackSize = MFI.getStackSize();
    2158             : 
    2159             :   // Do not generate a prologue for leaf functions with a stack of size zero.
    2160             :   // For non-leaf functions we have to allow for the possibility that the
    2161             :   // call is to a non-split function, as in PR37807.
    2162          64 :   if (StackSize == 0 && !MFI.hasTailCall())
    2163             :     return;
    2164             : 
    2165             :   // Use R4 and R5 as scratch registers.
    2166             :   // We save R4 and R5 before use and restore them before leaving the function.
    2167             :   unsigned ScratchReg0 = ARM::R4;
    2168             :   unsigned ScratchReg1 = ARM::R5;
    2169             :   uint64_t AlignedStackSize;
    2170             : 
    2171          56 :   MachineBasicBlock *PrevStackMBB = MF.CreateMachineBasicBlock();
    2172          56 :   MachineBasicBlock *PostStackMBB = MF.CreateMachineBasicBlock();
    2173          56 :   MachineBasicBlock *AllocMBB = MF.CreateMachineBasicBlock();
    2174          56 :   MachineBasicBlock *GetMBB = MF.CreateMachineBasicBlock();
    2175          56 :   MachineBasicBlock *McrMBB = MF.CreateMachineBasicBlock();
    2176             : 
    2177             :   // Grab everything that reaches PrologueMBB to update there liveness as well.
    2178             :   SmallPtrSet<MachineBasicBlock *, 8> BeforePrologueRegion;
    2179             :   SmallVector<MachineBasicBlock *, 2> WalkList;
    2180          56 :   WalkList.push_back(&PrologueMBB);
    2181             : 
    2182             :   do {
    2183             :     MachineBasicBlock *CurMBB = WalkList.pop_back_val();
    2184          56 :     for (MachineBasicBlock *PredBB : CurMBB->predecessors()) {
    2185           0 :       if (BeforePrologueRegion.insert(PredBB).second)
    2186           0 :         WalkList.push_back(PredBB);
    2187             :     }
    2188          56 :   } while (!WalkList.empty());
    2189             : 
    2190             :   // The order in that list is important.
    2191             :   // The blocks will all be inserted before PrologueMBB using that order.
    2192             :   // Therefore the block that should appear first in the CFG should appear
    2193             :   // first in the list.
    2194             :   MachineBasicBlock *AddedBlocks[] = {PrevStackMBB, McrMBB, GetMBB, AllocMBB,
    2195          56 :                                       PostStackMBB};
    2196             : 
    2197         336 :   for (MachineBasicBlock *B : AddedBlocks)
    2198         280 :     BeforePrologueRegion.insert(B);
    2199             : 
    2200         224 :   for (const auto &LI : PrologueMBB.liveins()) {
    2201        1008 :     for (MachineBasicBlock *PredBB : BeforePrologueRegion)
    2202             :       PredBB->addLiveIn(LI);
    2203             :   }
    2204             : 
    2205             :   // Remove the newly added blocks from the list, since we know
    2206             :   // we do not have to do the following updates for them.
    2207         336 :   for (MachineBasicBlock *B : AddedBlocks) {
    2208             :     BeforePrologueRegion.erase(B);
    2209         280 :     MF.insert(PrologueMBB.getIterator(), B);
    2210             :   }
    2211             : 
    2212          56 :   for (MachineBasicBlock *MBB : BeforePrologueRegion) {
    2213             :     // Make sure the LiveIns are still sorted and unique.
    2214           0 :     MBB->sortUniqueLiveIns();
    2215             :     // Replace the edges to PrologueMBB by edges to the sequences
    2216             :     // we are about to add.
    2217           0 :     MBB->ReplaceUsesOfBlockWith(&PrologueMBB, AddedBlocks[0]);
    2218             :   }
    2219             : 
    2220             :   // The required stack size that is aligned to ARM constant criterion.
    2221          56 :   AlignedStackSize = alignToARMConstant(StackSize);
    2222             : 
    2223             :   // When the frame size is less than 256 we just compare the stack
    2224             :   // boundary directly to the value of the stack pointer, per gcc.
    2225             :   bool CompareStackPointer = AlignedStackSize < kSplitStackAvailable;
    2226             : 
    2227             :   // We will use two of the callee save registers as scratch registers so we
    2228             :   // need to save those registers onto the stack.
    2229             :   // We will use SR0 to hold stack limit and SR1 to hold the stack size
    2230             :   // requested and arguments for __morestack().
    2231             :   // SR0: Scratch Register #0
    2232             :   // SR1: Scratch Register #1
    2233             :   // push {SR0, SR1}
    2234          56 :   if (Thumb) {
    2235          26 :     BuildMI(PrevStackMBB, DL, TII.get(ARM::tPUSH))
    2236          26 :         .add(predOps(ARMCC::AL))
    2237          26 :         .addReg(ScratchReg0)
    2238          26 :         .addReg(ScratchReg1);
    2239             :   } else {
    2240          30 :     BuildMI(PrevStackMBB, DL, TII.get(ARM::STMDB_UPD))
    2241          30 :         .addReg(ARM::SP, RegState::Define)
    2242          30 :         .addReg(ARM::SP)
    2243          30 :         .add(predOps(ARMCC::AL))
    2244          30 :         .addReg(ScratchReg0)
    2245          30 :         .addReg(ScratchReg1);
    2246             :   }
    2247             : 
    2248             :   // Emit the relevant DWARF information about the change in stack pointer as
    2249             :   // well as where to find both r4 and r5 (the callee-save registers)
    2250             :   CFIIndex =
    2251          56 :       MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, -8));
    2252          56 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2253             :       .addCFIIndex(CFIIndex);
    2254          56 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2255          56 :       nullptr, MRI->getDwarfRegNum(ScratchReg1, true), -4));
    2256          56 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2257             :       .addCFIIndex(CFIIndex);
    2258          56 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2259          56 :       nullptr, MRI->getDwarfRegNum(ScratchReg0, true), -8));
    2260          56 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2261             :       .addCFIIndex(CFIIndex);
    2262             : 
    2263             :   // mov SR1, sp
    2264          56 :   if (Thumb) {
    2265          26 :     BuildMI(McrMBB, DL, TII.get(ARM::tMOVr), ScratchReg1)
    2266          26 :         .addReg(ARM::SP)
    2267          26 :         .add(predOps(ARMCC::AL));
    2268          30 :   } else if (CompareStackPointer) {
    2269          22 :     BuildMI(McrMBB, DL, TII.get(ARM::MOVr), ScratchReg1)
    2270          22 :         .addReg(ARM::SP)
    2271          22 :         .add(predOps(ARMCC::AL))
    2272          22 :         .add(condCodeOp());
    2273             :   }
    2274             : 
    2275             :   // sub SR1, sp, #StackSize
    2276          56 :   if (!CompareStackPointer && Thumb) {
    2277           8 :     BuildMI(McrMBB, DL, TII.get(ARM::tSUBi8), ScratchReg1)
    2278           8 :         .add(condCodeOp())
    2279           8 :         .addReg(ScratchReg1)
    2280           8 :         .addImm(AlignedStackSize)
    2281           8 :         .add(predOps(ARMCC::AL));
    2282          48 :   } else if (!CompareStackPointer) {
    2283           8 :     BuildMI(McrMBB, DL, TII.get(ARM::SUBri), ScratchReg1)
    2284           8 :         .addReg(ARM::SP)
    2285           8 :         .addImm(AlignedStackSize)
    2286           8 :         .add(predOps(ARMCC::AL))
    2287           8 :         .add(condCodeOp());
    2288             :   }
    2289             : 
    2290          56 :   if (Thumb && ST->isThumb1Only()) {
    2291             :     unsigned PCLabelId = ARMFI->createPICLabelUId();
    2292          24 :     ARMConstantPoolValue *NewCPV = ARMConstantPoolSymbol::Create(
    2293          24 :         MF.getFunction().getContext(), "__STACK_LIMIT", PCLabelId, 0);
    2294          24 :     MachineConstantPool *MCP = MF.getConstantPool();
    2295          24 :     unsigned CPI = MCP->getConstantPoolIndex(NewCPV, 4);
    2296             : 
    2297             :     // ldr SR0, [pc, offset(STACK_LIMIT)]
    2298          24 :     BuildMI(GetMBB, DL, TII.get(ARM::tLDRpci), ScratchReg0)
    2299             :         .addConstantPoolIndex(CPI)
    2300          24 :         .add(predOps(ARMCC::AL));
    2301             : 
    2302             :     // ldr SR0, [SR0]
    2303          24 :     BuildMI(GetMBB, DL, TII.get(ARM::tLDRi), ScratchReg0)
    2304          24 :         .addReg(ScratchReg0)
    2305             :         .addImm(0)
    2306          24 :         .add(predOps(ARMCC::AL));
    2307             :   } else {
    2308             :     // Get TLS base address from the coprocessor
    2309             :     // mrc p15, #0, SR0, c13, c0, #3
    2310          32 :     BuildMI(McrMBB, DL, TII.get(ARM::MRC), ScratchReg0)
    2311             :         .addImm(15)
    2312             :         .addImm(0)
    2313             :         .addImm(13)
    2314             :         .addImm(0)
    2315             :         .addImm(3)
    2316          32 :         .add(predOps(ARMCC::AL));
    2317             : 
    2318             :     // Use the last tls slot on android and a private field of the TCP on linux.
    2319             :     assert(ST->isTargetAndroid() || ST->isTargetLinux());
    2320          32 :     unsigned TlsOffset = ST->isTargetAndroid() ? 63 : 1;
    2321             : 
    2322             :     // Get the stack limit from the right offset
    2323             :     // ldr SR0, [sr0, #4 * TlsOffset]
    2324          32 :     BuildMI(GetMBB, DL, TII.get(ARM::LDRi12), ScratchReg0)
    2325          32 :         .addReg(ScratchReg0)
    2326          32 :         .addImm(4 * TlsOffset)
    2327          32 :         .add(predOps(ARMCC::AL));
    2328             :   }
    2329             : 
    2330             :   // Compare stack limit with stack size requested.
    2331             :   // cmp SR0, SR1
    2332          56 :   Opcode = Thumb ? ARM::tCMPr : ARM::CMPrr;
    2333          56 :   BuildMI(GetMBB, DL, TII.get(Opcode))
    2334          56 :       .addReg(ScratchReg0)
    2335          56 :       .addReg(ScratchReg1)
    2336          56 :       .add(predOps(ARMCC::AL));
    2337             : 
    2338             :   // This jump is taken if StackLimit < SP - stack required.
    2339          56 :   Opcode = Thumb ? ARM::tBcc : ARM::Bcc;
    2340          56 :   BuildMI(GetMBB, DL, TII.get(Opcode)).addMBB(PostStackMBB)
    2341             :        .addImm(ARMCC::LO)
    2342          56 :        .addReg(ARM::CPSR);
    2343             : 
    2344             : 
    2345             :   // Calling __morestack(StackSize, Size of stack arguments).
    2346             :   // __morestack knows that the stack size requested is in SR0(r4)
    2347             :   // and amount size of stack arguments is in SR1(r5).
    2348             : 
    2349             :   // Pass first argument for the __morestack by Scratch Register #0.
    2350             :   //   The amount size of stack required
    2351          56 :   if (Thumb) {
    2352          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tMOVi8), ScratchReg0)
    2353          26 :         .add(condCodeOp())
    2354          26 :         .addImm(AlignedStackSize)
    2355          26 :         .add(predOps(ARMCC::AL));
    2356             :   } else {
    2357          30 :     BuildMI(AllocMBB, DL, TII.get(ARM::MOVi), ScratchReg0)
    2358          30 :         .addImm(AlignedStackSize)
    2359          30 :         .add(predOps(ARMCC::AL))
    2360          30 :         .add(condCodeOp());
    2361             :   }
    2362             :   // Pass second argument for the __morestack by Scratch Register #1.
    2363             :   //   The amount size of stack consumed to save function arguments.
    2364          56 :   if (Thumb) {
    2365          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tMOVi8), ScratchReg1)
    2366          26 :         .add(condCodeOp())
    2367          26 :         .addImm(alignToARMConstant(ARMFI->getArgumentStackSize()))
    2368          26 :         .add(predOps(ARMCC::AL));
    2369             :   } else {
    2370          30 :     BuildMI(AllocMBB, DL, TII.get(ARM::MOVi), ScratchReg1)
    2371          30 :         .addImm(alignToARMConstant(ARMFI->getArgumentStackSize()))
    2372          30 :         .add(predOps(ARMCC::AL))
    2373          30 :         .add(condCodeOp());
    2374             :   }
    2375             : 
    2376             :   // push {lr} - Save return address of this function.
    2377          56 :   if (Thumb) {
    2378          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tPUSH))
    2379          26 :         .add(predOps(ARMCC::AL))
    2380          26 :         .addReg(ARM::LR);
    2381             :   } else {
    2382          30 :     BuildMI(AllocMBB, DL, TII.get(ARM::STMDB_UPD))
    2383          30 :         .addReg(ARM::SP, RegState::Define)
    2384          30 :         .addReg(ARM::SP)
    2385          30 :         .add(predOps(ARMCC::AL))
    2386          30 :         .addReg(ARM::LR);
    2387             :   }
    2388             : 
    2389             :   // Emit the DWARF info about the change in stack as well as where to find the
    2390             :   // previous link register
    2391             :   CFIIndex =
    2392          56 :       MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, -12));
    2393          56 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2394             :       .addCFIIndex(CFIIndex);
    2395          56 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2396          56 :         nullptr, MRI->getDwarfRegNum(ARM::LR, true), -12));
    2397          56 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2398             :       .addCFIIndex(CFIIndex);
    2399             : 
    2400             :   // Call __morestack().
    2401          56 :   if (Thumb) {
    2402          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tBL))
    2403          26 :         .add(predOps(ARMCC::AL))
    2404             :         .addExternalSymbol("__morestack");
    2405             :   } else {
    2406          30 :     BuildMI(AllocMBB, DL, TII.get(ARM::BL))
    2407             :         .addExternalSymbol("__morestack");
    2408             :   }
    2409             : 
    2410             :   // pop {lr} - Restore return address of this original function.
    2411          56 :   if (Thumb) {
    2412          26 :     if (ST->isThumb1Only()) {
    2413          24 :       BuildMI(AllocMBB, DL, TII.get(ARM::tPOP))
    2414          24 :           .add(predOps(ARMCC::AL))
    2415          24 :           .addReg(ScratchReg0);
    2416          24 :       BuildMI(AllocMBB, DL, TII.get(ARM::tMOVr), ARM::LR)
    2417          24 :           .addReg(ScratchReg0)
    2418          24 :           .add(predOps(ARMCC::AL));
    2419             :     } else {
    2420           2 :       BuildMI(AllocMBB, DL, TII.get(ARM::t2LDR_POST))
    2421           2 :           .addReg(ARM::LR, RegState::Define)
    2422           2 :           .addReg(ARM::SP, RegState::Define)
    2423           2 :           .addReg(ARM::SP)
    2424             :           .addImm(4)
    2425           2 :           .add(predOps(ARMCC::AL));
    2426             :     }
    2427             :   } else {
    2428          30 :     BuildMI(AllocMBB, DL, TII.get(ARM::LDMIA_UPD))
    2429          30 :         .addReg(ARM::SP, RegState::Define)
    2430          30 :         .addReg(ARM::SP)
    2431          30 :         .add(predOps(ARMCC::AL))
    2432          30 :         .addReg(ARM::LR);
    2433             :   }
    2434             : 
    2435             :   // Restore SR0 and SR1 in case of __morestack() was called.
    2436             :   // __morestack() will skip PostStackMBB block so we need to restore
    2437             :   // scratch registers from here.
    2438             :   // pop {SR0, SR1}
    2439          56 :   if (Thumb) {
    2440          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tPOP))
    2441          26 :         .add(predOps(ARMCC::AL))
    2442          26 :         .addReg(ScratchReg0)
    2443          26 :         .addReg(ScratchReg1);
    2444             :   } else {
    2445          30 :     BuildMI(AllocMBB, DL, TII.get(ARM::LDMIA_UPD))
    2446          30 :         .addReg(ARM::SP, RegState::Define)
    2447          30 :         .addReg(ARM::SP)
    2448          30 :         .add(predOps(ARMCC::AL))
    2449          30 :         .addReg(ScratchReg0)
    2450          30 :         .addReg(ScratchReg1);
    2451             :   }
    2452             : 
    2453             :   // Update the CFA offset now that we've popped
    2454          56 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, 0));
    2455          56 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2456             :       .addCFIIndex(CFIIndex);
    2457             : 
    2458             :   // Return from this function.
    2459          56 :   BuildMI(AllocMBB, DL, TII.get(ST->getReturnOpcode())).add(predOps(ARMCC::AL));
    2460             : 
    2461             :   // Restore SR0 and SR1 in case of __morestack() was not called.
    2462             :   // pop {SR0, SR1}
    2463          56 :   if (Thumb) {
    2464          26 :     BuildMI(PostStackMBB, DL, TII.get(ARM::tPOP))
    2465          26 :         .add(predOps(ARMCC::AL))
    2466          26 :         .addReg(ScratchReg0)
    2467          26 :         .addReg(ScratchReg1);
    2468             :   } else {
    2469          30 :     BuildMI(PostStackMBB, DL, TII.get(ARM::LDMIA_UPD))
    2470          30 :         .addReg(ARM::SP, RegState::Define)
    2471          30 :         .addReg(ARM::SP)
    2472          30 :         .add(predOps(ARMCC::AL))
    2473          30 :         .addReg(ScratchReg0)
    2474          30 :         .addReg(ScratchReg1);
    2475             :   }
    2476             : 
    2477             :   // Update the CFA offset now that we've popped
    2478          56 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, 0));
    2479          56 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2480             :       .addCFIIndex(CFIIndex);
    2481             : 
    2482             :   // Tell debuggers that r4 and r5 are now the same as they were in the
    2483             :   // previous function, that they're the "Same Value".
    2484          56 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
    2485          56 :       nullptr, MRI->getDwarfRegNum(ScratchReg0, true)));
    2486          56 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2487             :       .addCFIIndex(CFIIndex);
    2488          56 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
    2489          56 :       nullptr, MRI->getDwarfRegNum(ScratchReg1, true)));
    2490          56 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2491             :       .addCFIIndex(CFIIndex);
    2492             : 
    2493             :   // Organizing MBB lists
    2494          56 :   PostStackMBB->addSuccessor(&PrologueMBB);
    2495             : 
    2496          56 :   AllocMBB->addSuccessor(PostStackMBB);
    2497             : 
    2498          56 :   GetMBB->addSuccessor(PostStackMBB);
    2499          56 :   GetMBB->addSuccessor(AllocMBB);
    2500             : 
    2501          56 :   McrMBB->addSuccessor(GetMBB);
    2502             : 
    2503          56 :   PrevStackMBB->addSuccessor(McrMBB);
    2504             : 
    2505             : #ifdef EXPENSIVE_CHECKS
    2506             :   MF.verify();
    2507             : #endif
    2508             : }

Generated by: LCOV version 1.13