LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMFrameLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 971 993 97.8 %
Date: 2018-06-17 00:07:59 Functions: 32 32 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- ARMFrameLowering.cpp - ARM Frame Information -----------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the ARM implementation of TargetFrameLowering class.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "ARMFrameLowering.h"
      15             : #include "ARMBaseInstrInfo.h"
      16             : #include "ARMBaseRegisterInfo.h"
      17             : #include "ARMConstantPoolValue.h"
      18             : #include "ARMMachineFunctionInfo.h"
      19             : #include "ARMSubtarget.h"
      20             : #include "MCTargetDesc/ARMAddressingModes.h"
      21             : #include "MCTargetDesc/ARMBaseInfo.h"
      22             : #include "Utils/ARMBaseInfo.h"
      23             : #include "llvm/ADT/BitVector.h"
      24             : #include "llvm/ADT/STLExtras.h"
      25             : #include "llvm/ADT/SmallPtrSet.h"
      26             : #include "llvm/ADT/SmallVector.h"
      27             : #include "llvm/CodeGen/MachineBasicBlock.h"
      28             : #include "llvm/CodeGen/MachineConstantPool.h"
      29             : #include "llvm/CodeGen/MachineFrameInfo.h"
      30             : #include "llvm/CodeGen/MachineFunction.h"
      31             : #include "llvm/CodeGen/MachineInstr.h"
      32             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      33             : #include "llvm/CodeGen/MachineModuleInfo.h"
      34             : #include "llvm/CodeGen/MachineOperand.h"
      35             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      36             : #include "llvm/CodeGen/RegisterScavenging.h"
      37             : #include "llvm/CodeGen/TargetInstrInfo.h"
      38             : #include "llvm/CodeGen/TargetOpcodes.h"
      39             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      40             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      41             : #include "llvm/IR/Attributes.h"
      42             : #include "llvm/IR/CallingConv.h"
      43             : #include "llvm/IR/DebugLoc.h"
      44             : #include "llvm/IR/Function.h"
      45             : #include "llvm/MC/MCContext.h"
      46             : #include "llvm/MC/MCDwarf.h"
      47             : #include "llvm/MC/MCInstrDesc.h"
      48             : #include "llvm/MC/MCRegisterInfo.h"
      49             : #include "llvm/Support/CodeGen.h"
      50             : #include "llvm/Support/CommandLine.h"
      51             : #include "llvm/Support/Compiler.h"
      52             : #include "llvm/Support/Debug.h"
      53             : #include "llvm/Support/ErrorHandling.h"
      54             : #include "llvm/Support/MathExtras.h"
      55             : #include "llvm/Support/raw_ostream.h"
      56             : #include "llvm/Target/TargetMachine.h"
      57             : #include "llvm/Target/TargetOptions.h"
      58             : #include <algorithm>
      59             : #include <cassert>
      60             : #include <cstddef>
      61             : #include <cstdint>
      62             : #include <iterator>
      63             : #include <utility>
      64             : #include <vector>
      65             : 
      66             : #define DEBUG_TYPE "arm-frame-lowering"
      67             : 
      68             : using namespace llvm;
      69             : 
      70             : static cl::opt<bool>
      71      303507 : SpillAlignedNEONRegs("align-neon-spills", cl::Hidden, cl::init(true),
      72      202338 :                      cl::desc("Align ARM NEON spills in prolog and epilog"));
      73             : 
      74             : static MachineBasicBlock::iterator
      75             : skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
      76             :                         unsigned NumAlignedDPRCS2Regs);
      77             : 
      78        4804 : ARMFrameLowering::ARMFrameLowering(const ARMSubtarget &sti)
      79             :     : TargetFrameLowering(StackGrowsDown, sti.getStackAlignment(), 0, 4),
      80        9608 :       STI(sti) {}
      81             : 
      82      339254 : bool ARMFrameLowering::noFramePointerElim(const MachineFunction &MF) const {
      83             :   // iOS always has a FP for backtracking, force other targets to keep their FP
      84             :   // when doing FastISel. The emitted code is currently superior, and in cases
      85             :   // like test-suite's lencod FastISel isn't quite correct when FP is eliminated.
      86      645027 :   return TargetFrameLowering::noFramePointerElim(MF) ||
      87      645027 :          MF.getSubtarget<ARMSubtarget>().useFastISel();
      88             : }
      89             : 
      90             : /// Returns true if the target can safely skip saving callee-saved registers
      91             : /// for noreturn nounwind functions.
      92          14 : bool ARMFrameLowering::enableCalleeSaveSkip(const MachineFunction &MF) const {
      93             :   assert(MF.getFunction().hasFnAttribute(Attribute::NoReturn) &&
      94             :          MF.getFunction().hasFnAttribute(Attribute::NoUnwind) &&
      95             :          !MF.getFunction().hasFnAttribute(Attribute::UWTable));
      96             : 
      97             :   // Frame pointer and link register are not treated as normal CSR, thus we
      98             :   // can always skip CSR saves for nonreturning functions.
      99          14 :   return true;
     100             : }
     101             : 
     102             : /// hasFP - Return true if the specified function should have a dedicated frame
     103             : /// pointer register.  This is true if the function has variable sized allocas
     104             : /// or if frame pointer elimination is disabled.
     105      140328 : bool ARMFrameLowering::hasFP(const MachineFunction &MF) const {
     106      140328 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     107      140328 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     108             : 
     109             :   // ABI-required frame pointer.
     110      140328 :   if (MF.getTarget().Options.DisableFramePointerElim(MF))
     111             :     return true;
     112             : 
     113             :   // Frame pointer required for use within this function.
     114      233089 :   return (RegInfo->needsStackRealignment(MF) ||
     115      232317 :           MFI.hasVarSizedObjects() ||
     116      114529 :           MFI.isFrameAddressTaken());
     117             : }
     118             : 
     119             : /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
     120             : /// not required, we reserve argument space for call sites in the function
     121             : /// immediately on entry to the current function.  This eliminates the need for
     122             : /// add/sub sp brackets around call sites.  Returns true if the call frame is
     123             : /// included as part of the stack frame.
     124       61366 : bool ARMFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
     125       61366 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     126             :   unsigned CFSize = MFI.getMaxCallFrameSize();
     127             :   // It's not always a good idea to include the call frame as part of the
     128             :   // stack frame. ARM (especially Thumb) has small immediate offset to
     129             :   // address the stack frame. So a large call frame can cause poor codegen
     130             :   // and may even makes it impossible to scavenge a register.
     131       61365 :   if (CFSize >= ((1 << 12) - 1) / 2)  // Half of imm12
     132             :     return false;
     133             : 
     134       60153 :   return !MFI.hasVarSizedObjects();
     135             : }
     136             : 
     137             : /// canSimplifyCallFramePseudos - If there is a reserved call frame, the
     138             : /// call frame pseudos can be simplified.  Unlike most targets, having a FP
     139             : /// is not sufficient here since we still may reference some objects via SP
     140             : /// even when FP is available in Thumb2 mode.
     141             : bool
     142       18337 : ARMFrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
     143       18337 :   return hasReservedCallFrame(MF) || MF.getFrameInfo().hasVarSizedObjects();
     144             : }
     145             : 
     146        7731 : static bool isCSRestore(MachineInstr &MI, const ARMBaseInstrInfo &TII,
     147             :                         const MCPhysReg *CSRegs) {
     148             :   // Integer spill area is handled with "pop".
     149       15462 :   if (isPopOpcode(MI.getOpcode())) {
     150             :     // The first two operands are predicates. The last two are
     151             :     // imp-def and imp-use of SP. Check everything in between.
     152        3824 :     for (int i = 5, e = MI.getNumOperands(); i != e; ++i)
     153        7752 :       if (!isCalleeSavedRegister(MI.getOperand(i).getReg(), CSRegs))
     154             :         return false;
     155             :     return true;
     156             :   }
     157        6299 :   if ((MI.getOpcode() == ARM::LDR_POST_IMM ||
     158        6299 :        MI.getOpcode() == ARM::LDR_POST_REG ||
     159         266 :        MI.getOpcode() == ARM::t2LDR_POST) &&
     160        7023 :       isCalleeSavedRegister(MI.getOperand(0).getReg(), CSRegs) &&
     161         266 :       MI.getOperand(1).getReg() == ARM::SP)
     162             :     return true;
     163             : 
     164             :   return false;
     165             : }
     166             : 
     167        3695 : static void emitRegPlusImmediate(
     168             :     bool isARM, MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
     169             :     const DebugLoc &dl, const ARMBaseInstrInfo &TII, unsigned DestReg,
     170             :     unsigned SrcReg, int NumBytes, unsigned MIFlags = MachineInstr::NoFlags,
     171             :     ARMCC::CondCodes Pred = ARMCC::AL, unsigned PredReg = 0) {
     172        3695 :   if (isARM)
     173        2382 :     emitARMRegPlusImmediate(MBB, MBBI, dl, DestReg, SrcReg, NumBytes,
     174             :                             Pred, PredReg, TII, MIFlags);
     175             :   else
     176        1313 :     emitT2RegPlusImmediate(MBB, MBBI, dl, DestReg, SrcReg, NumBytes,
     177             :                            Pred, PredReg, TII, MIFlags);
     178        3695 : }
     179             : 
     180             : static void emitSPUpdate(bool isARM, MachineBasicBlock &MBB,
     181             :                          MachineBasicBlock::iterator &MBBI, const DebugLoc &dl,
     182             :                          const ARMBaseInstrInfo &TII, int NumBytes,
     183             :                          unsigned MIFlags = MachineInstr::NoFlags,
     184             :                          ARMCC::CondCodes Pred = ARMCC::AL,
     185             :                          unsigned PredReg = 0) {
     186        2733 :   emitRegPlusImmediate(isARM, MBB, MBBI, dl, TII, ARM::SP, ARM::SP, NumBytes,
     187             :                        MIFlags, Pred, PredReg);
     188             : }
     189             : 
     190        1228 : static int sizeOfSPAdjustment(const MachineInstr &MI) {
     191             :   int RegSize;
     192        2456 :   switch (MI.getOpcode()) {
     193             :   case ARM::VSTMDDB_UPD:
     194             :     RegSize = 8;
     195             :     break;
     196         954 :   case ARM::STMDB_UPD:
     197             :   case ARM::t2STMDB_UPD:
     198             :     RegSize = 4;
     199         954 :     break;
     200             :   case ARM::t2STR_PRE:
     201             :   case ARM::STR_PRE_IMM:
     202             :     return 4;
     203           0 :   default:
     204           0 :     llvm_unreachable("Unknown push or pop like instruction");
     205             :   }
     206             : 
     207             :   int count = 0;
     208             :   // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
     209             :   // pred) so the list starts at 4.
     210        5418 :   for (int i = MI.getNumOperands() - 1; i >= 4; --i)
     211        4198 :     count += RegSize;
     212             :   return count;
     213             : }
     214             : 
     215         325 : static bool WindowsRequiresStackProbe(const MachineFunction &MF,
     216             :                                       size_t StackSizeInBytes) {
     217         325 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     218         325 :   const Function &F = MF.getFunction();
     219         325 :   unsigned StackProbeSize = (MFI.getStackProtectorIndex() > 0) ? 4080 : 4096;
     220         325 :   if (F.hasFnAttribute("stack-probe-size"))
     221           8 :     F.getFnAttribute("stack-probe-size")
     222           8 :         .getValueAsString()
     223           4 :         .getAsInteger(0, StackProbeSize);
     224         333 :   return (StackSizeInBytes >= StackProbeSize) &&
     225         325 :          !F.hasFnAttribute("no-stack-arg-probe");
     226             : }
     227             : 
     228             : namespace {
     229             : 
     230       12546 : struct StackAdjustingInsts {
     231             :   struct InstInfo {
     232             :     MachineBasicBlock::iterator I;
     233             :     unsigned SPAdjust;
     234             :     bool BeforeFPSet;
     235             :   };
     236             : 
     237             :   SmallVector<InstInfo, 4> Insts;
     238             : 
     239             :   void addInst(MachineBasicBlock::iterator I, unsigned SPAdjust,
     240             :                bool BeforeFPSet = false) {
     241        5443 :     InstInfo Info = {I, SPAdjust, BeforeFPSet};
     242        5443 :     Insts.push_back(Info);
     243             :   }
     244             : 
     245             :   void addExtraBytes(const MachineBasicBlock::iterator I, unsigned ExtraBytes) {
     246             :     auto Info =
     247             :         llvm::find_if(Insts, [&](InstInfo &Info) { return Info.I == I; });
     248             :     assert(Info != Insts.end() && "invalid sp adjusting instruction");
     249          23 :     Info->SPAdjust += ExtraBytes;
     250             :   }
     251             : 
     252       12546 :   void emitDefCFAOffsets(MachineBasicBlock &MBB, const DebugLoc &dl,
     253             :                          const ARMBaseInstrInfo &TII, bool HasFP) {
     254       12546 :     MachineFunction &MF = *MBB.getParent();
     255             :     unsigned CFAOffset = 0;
     256       22080 :     for (auto &Info : Insts) {
     257        5311 :       if (HasFP && !Info.BeforeFPSet)
     258             :         return;
     259             : 
     260        4767 :       CFAOffset -= Info.SPAdjust;
     261             :       unsigned CFIIndex = MF.addFrameInst(
     262        9534 :           MCCFIInstruction::createDefCfaOffset(nullptr, CFAOffset));
     263        9534 :       BuildMI(MBB, std::next(Info.I), dl,
     264        4767 :               TII.get(TargetOpcode::CFI_INSTRUCTION))
     265             :               .addCFIIndex(CFIIndex)
     266             :               .setMIFlags(MachineInstr::FrameSetup);
     267             :     }
     268             :   }
     269             : };
     270             : 
     271             : } // end anonymous namespace
     272             : 
     273             : /// Emit an instruction sequence that will align the address in
     274             : /// register Reg by zero-ing out the lower bits.  For versions of the
     275             : /// architecture that support Neon, this must be done in a single
     276             : /// instruction, since skipAlignedDPRCS2Spills assumes it is done in a
     277             : /// single instruction. That function only gets called when optimizing
     278             : /// spilling of D registers on a core with the Neon instruction set
     279             : /// present.
     280         204 : static void emitAligningInstructions(MachineFunction &MF, ARMFunctionInfo *AFI,
     281             :                                      const TargetInstrInfo &TII,
     282             :                                      MachineBasicBlock &MBB,
     283             :                                      MachineBasicBlock::iterator MBBI,
     284             :                                      const DebugLoc &DL, const unsigned Reg,
     285             :                                      const unsigned Alignment,
     286             :                                      const bool MustBeSingleInstruction) {
     287             :   const ARMSubtarget &AST =
     288         204 :       static_cast<const ARMSubtarget &>(MF.getSubtarget());
     289         204 :   const bool CanUseBFC = AST.hasV6T2Ops() || AST.hasV7Ops();
     290         204 :   const unsigned AlignMask = Alignment - 1;
     291         204 :   const unsigned NrBitsToZero = countTrailingZeros(Alignment);
     292             :   assert(!AFI->isThumb1OnlyFunction() && "Thumb1 not supported");
     293         204 :   if (!AFI->isThumbFunction()) {
     294             :     // if the BFC instruction is available, use that to zero the lower
     295             :     // bits:
     296             :     //   bfc Reg, #0, log2(Alignment)
     297             :     // otherwise use BIC, if the mask to zero the required number of bits
     298             :     // can be encoded in the bic immediate field
     299             :     //   bic Reg, Reg, Alignment-1
     300             :     // otherwise, emit
     301             :     //   lsr Reg, Reg, log2(Alignment)
     302             :     //   lsl Reg, Reg, log2(Alignment)
     303         114 :     if (CanUseBFC) {
     304         297 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::BFC), Reg)
     305          99 :           .addReg(Reg, RegState::Kill)
     306          99 :           .addImm(~AlignMask)
     307          99 :           .add(predOps(ARMCC::AL));
     308          15 :     } else if (AlignMask <= 255) {
     309          39 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::BICri), Reg)
     310          13 :           .addReg(Reg, RegState::Kill)
     311          13 :           .addImm(AlignMask)
     312          13 :           .add(predOps(ARMCC::AL))
     313          13 :           .add(condCodeOp());
     314             :     } else {
     315             :       assert(!MustBeSingleInstruction &&
     316             :              "Shouldn't call emitAligningInstructions demanding a single "
     317             :              "instruction to be emitted for large stack alignment for a target "
     318             :              "without BFC.");
     319           6 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::MOVsi), Reg)
     320           2 :           .addReg(Reg, RegState::Kill)
     321           2 :           .addImm(ARM_AM::getSORegOpc(ARM_AM::lsr, NrBitsToZero))
     322           2 :           .add(predOps(ARMCC::AL))
     323           2 :           .add(condCodeOp());
     324           6 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::MOVsi), Reg)
     325           2 :           .addReg(Reg, RegState::Kill)
     326           2 :           .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, NrBitsToZero))
     327           2 :           .add(predOps(ARMCC::AL))
     328           2 :           .add(condCodeOp());
     329             :     }
     330             :   } else {
     331             :     // Since this is only reached for Thumb-2 targets, the BFC instruction
     332             :     // should always be available.
     333             :     assert(CanUseBFC);
     334         270 :     BuildMI(MBB, MBBI, DL, TII.get(ARM::t2BFC), Reg)
     335          90 :         .addReg(Reg, RegState::Kill)
     336          90 :         .addImm(~AlignMask)
     337          90 :         .add(predOps(ARMCC::AL));
     338             :   }
     339         204 : }
     340             : 
     341             : /// We need the offset of the frame pointer relative to other MachineFrameInfo
     342             : /// offsets which are encoded relative to SP at function begin.
     343             : /// See also emitPrologue() for how the FP is set up.
     344             : /// Unfortunately we cannot determine this value in determineCalleeSaves() yet
     345             : /// as assignCalleeSavedSpillSlots() hasn't run at this point. Instead we use
     346             : /// this to produce a conservative estimate that we check in an assert() later.
     347             : static int getMaxFPOffset(const Function &F, const ARMFunctionInfo &AFI) {
     348             :   // This is a conservative estimation: Assume the frame pointer being r7 and
     349             :   // pc("r15") up to r8 getting spilled before (= 8 registers).
     350       13666 :   return -AFI.getArgRegsSaveSize() - (8 * 4);
     351             : }
     352             : 
     353       12548 : void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     354             :                                     MachineBasicBlock &MBB) const {
     355       12548 :   MachineBasicBlock::iterator MBBI = MBB.begin();
     356       12548 :   MachineFrameInfo  &MFI = MF.getFrameInfo();
     357       12548 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     358       12548 :   MachineModuleInfo &MMI = MF.getMMI();
     359             :   MCContext &Context = MMI.getContext();
     360       12548 :   const TargetMachine &TM = MF.getTarget();
     361       12548 :   const MCRegisterInfo *MRI = Context.getRegisterInfo();
     362       12548 :   const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
     363       12548 :   const ARMBaseInstrInfo &TII = *STI.getInstrInfo();
     364             :   assert(!AFI->isThumb1OnlyFunction() &&
     365             :          "This emitPrologue does not support Thumb1!");
     366       12548 :   bool isARM = !AFI->isThumbFunction();
     367       12548 :   unsigned Align = STI.getFrameLowering()->getStackAlignment();
     368       12548 :   unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
     369       12548 :   unsigned NumBytes = MFI.getStackSize();
     370             :   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
     371             : 
     372             :   // Debug location must be unknown since the first debug location is used
     373             :   // to determine the end of the prologue.
     374       12548 :   DebugLoc dl;
     375             :   
     376       12548 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
     377             : 
     378             :   // Determine the sizes of each callee-save spill areas and record which frame
     379             :   // belongs to which callee-save spill areas.
     380             :   unsigned GPRCS1Size = 0, GPRCS2Size = 0, DPRCSSize = 0;
     381             :   int FramePtrSpillFI = 0;
     382             :   int D8SpillFI = 0;
     383             : 
     384             :   // All calls are tail calls in GHC calling conv, and functions have no
     385             :   // prologue/epilogue.
     386       25096 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     387             :     return;
     388             : 
     389             :   StackAdjustingInsts DefCFAOffsetCandidates;
     390       12546 :   bool HasFP = hasFP(MF);
     391             : 
     392             :   // Allocate the vararg register save area.
     393       12546 :   if (ArgRegsSaveSize) {
     394          44 :     emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize,
     395             :                  MachineInstr::FrameSetup);
     396             :     DefCFAOffsetCandidates.addInst(std::prev(MBBI), ArgRegsSaveSize, true);
     397             :   }
     398             : 
     399       21555 :   if (!AFI->hasStackFrame() &&
     400        9071 :       (!STI.isTargetWindows() || !WindowsRequiresStackProbe(MF, NumBytes))) {
     401        9009 :     if (NumBytes - ArgRegsSaveSize != 0) {
     402         388 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -(NumBytes - ArgRegsSaveSize),
     403             :                    MachineInstr::FrameSetup);
     404         388 :       DefCFAOffsetCandidates.addInst(std::prev(MBBI),
     405             :                                      NumBytes - ArgRegsSaveSize, true);
     406             :     }
     407        9009 :     DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
     408             :     return;
     409             :   }
     410             : 
     411             :   // Determine spill area sizes.
     412       33261 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     413       22650 :     unsigned Reg = CSI[i].getReg();
     414       11325 :     int FI = CSI[i].getFrameIdx();
     415       11325 :     switch (Reg) {
     416        2112 :     case ARM::R8:
     417             :     case ARM::R9:
     418             :     case ARM::R10:
     419             :     case ARM::R11:
     420             :     case ARM::R12:
     421        2112 :       if (STI.splitFramePushPop(MF)) {
     422         212 :         GPRCS2Size += 4;
     423         212 :         break;
     424             :       }
     425             :       LLVM_FALLTHROUGH;
     426             :     case ARM::R0:
     427             :     case ARM::R1:
     428             :     case ARM::R2:
     429             :     case ARM::R3:
     430             :     case ARM::R4:
     431             :     case ARM::R5:
     432             :     case ARM::R6:
     433             :     case ARM::R7:
     434             :     case ARM::LR:
     435        9810 :       if (Reg == FramePtr)
     436             :         FramePtrSpillFI = FI;
     437        9810 :       GPRCS1Size += 4;
     438        9810 :       break;
     439        1303 :     default:
     440             :       // This is a DPR. Exclude the aligned DPRCS2 spills.
     441        1303 :       if (Reg == ARM::D8)
     442             :         D8SpillFI = FI;
     443        1303 :       if (Reg < ARM::D8 || Reg >= ARM::D8 + AFI->getNumAlignedDPRCS2Regs())
     444        1031 :         DPRCSSize += 8;
     445             :     }
     446             :   }
     447             : 
     448             :   // Move past area 1.
     449             :   MachineBasicBlock::iterator LastPush = MBB.end(), GPRCS1Push, GPRCS2Push;
     450        3537 :   if (GPRCS1Size > 0) {
     451             :     GPRCS1Push = LastPush = MBBI++;
     452             :     DefCFAOffsetCandidates.addInst(LastPush, GPRCS1Size, true);
     453             :   }
     454             : 
     455             :   // Determine starting offsets of spill areas.
     456        3537 :   unsigned GPRCS1Offset = NumBytes - ArgRegsSaveSize - GPRCS1Size;
     457        3537 :   unsigned GPRCS2Offset = GPRCS1Offset - GPRCS2Size;
     458        3777 :   unsigned DPRAlign = DPRCSSize ? std::min(8U, Align) : 4U;
     459        3537 :   unsigned DPRGapSize = (GPRCS1Size + GPRCS2Size + ArgRegsSaveSize) % DPRAlign;
     460        3537 :   unsigned DPRCSOffset = GPRCS2Offset - DPRGapSize - DPRCSSize;
     461             :   int FramePtrOffsetInPush = 0;
     462        3537 :   if (HasFP) {
     463             :     int FPOffset = MFI.getObjectOffset(FramePtrSpillFI);
     464             :     assert(getMaxFPOffset(MF.getFunction(), *AFI) <= FPOffset &&
     465             :            "Max FP estimation is wrong");
     466         962 :     FramePtrOffsetInPush = FPOffset + ArgRegsSaveSize;
     467         962 :     AFI->setFramePtrSpillOffset(MFI.getObjectOffset(FramePtrSpillFI) +
     468             :                                 NumBytes);
     469             :   }
     470             :   AFI->setGPRCalleeSavedArea1Offset(GPRCS1Offset);
     471             :   AFI->setGPRCalleeSavedArea2Offset(GPRCS2Offset);
     472             :   AFI->setDPRCalleeSavedAreaOffset(DPRCSOffset);
     473             : 
     474             :   // Move past area 2.
     475        3537 :   if (GPRCS2Size > 0) {
     476             :     GPRCS2Push = LastPush = MBBI++;
     477             :     DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size);
     478             :   }
     479             : 
     480             :   // Prolog/epilog inserter assumes we correctly align DPRs on the stack, so our
     481             :   // .cfi_offset operations will reflect that.
     482        3537 :   if (DPRGapSize) {
     483             :     assert(DPRGapSize == 4 && "unexpected alignment requirements for DPRs");
     484          29 :     if (LastPush != MBB.end() &&
     485          14 :         tryFoldSPUpdateIntoPushPop(STI, MF, &*LastPush, DPRGapSize))
     486             :       DefCFAOffsetCandidates.addExtraBytes(LastPush, DPRGapSize);
     487             :     else {
     488          13 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -DPRGapSize,
     489             :                    MachineInstr::FrameSetup);
     490             :       DefCFAOffsetCandidates.addInst(std::prev(MBBI), DPRGapSize);
     491             :     }
     492             :   }
     493             : 
     494             :   // Move past area 3.
     495        3537 :   if (DPRCSSize > 0) {
     496             :     // Since vpush register list cannot have gaps, there may be multiple vpush
     497             :     // instructions in the prologue.
     498        1277 :     while (MBBI != MBB.end() && MBBI->getOpcode() == ARM::VSTMDDB_UPD) {
     499         266 :       DefCFAOffsetCandidates.addInst(MBBI, sizeOfSPAdjustment(*MBBI));
     500             :       LastPush = MBBI++;
     501             :     }
     502             :   }
     503             : 
     504             :   // Move past the aligned DPRCS2 area.
     505        3537 :   if (AFI->getNumAlignedDPRCS2Regs() > 0) {
     506          40 :     MBBI = skipAlignedDPRCS2Spills(MBBI, AFI->getNumAlignedDPRCS2Regs());
     507             :     // The code inserted by emitAlignedDPRCS2Spills realigns the stack, and
     508             :     // leaves the stack pointer pointing to the DPRCS2 area.
     509             :     //
     510             :     // Adjust NumBytes to represent the stack slots below the DPRCS2 area.
     511          40 :     NumBytes += MFI.getObjectOffset(D8SpillFI);
     512             :   } else
     513             :     NumBytes = DPRCSOffset;
     514             : 
     515        7074 :   if (STI.isTargetWindows() && WindowsRequiresStackProbe(MF, NumBytes)) {
     516           3 :     uint32_t NumWords = NumBytes >> 2;
     517             : 
     518           3 :     if (NumWords < 65536)
     519           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi16), ARM::R4)
     520           3 :           .addImm(NumWords)
     521             :           .setMIFlags(MachineInstr::FrameSetup)
     522           3 :           .add(predOps(ARMCC::AL));
     523             :     else
     524           0 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi32imm), ARM::R4)
     525           0 :         .addImm(NumWords)
     526             :         .setMIFlags(MachineInstr::FrameSetup);
     527             : 
     528           3 :     switch (TM.getCodeModel()) {
     529           1 :     case CodeModel::Small:
     530             :     case CodeModel::Medium:
     531             :     case CodeModel::Kernel:
     532           3 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tBL))
     533           1 :           .add(predOps(ARMCC::AL))
     534             :           .addExternalSymbol("__chkstk")
     535           1 :           .addReg(ARM::R4, RegState::Implicit)
     536             :           .setMIFlags(MachineInstr::FrameSetup);
     537           1 :       break;
     538           2 :     case CodeModel::Large:
     539           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi32imm), ARM::R12)
     540             :         .addExternalSymbol("__chkstk")
     541             :         .setMIFlags(MachineInstr::FrameSetup);
     542             : 
     543           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tBLXr))
     544           2 :           .add(predOps(ARMCC::AL))
     545           2 :           .addReg(ARM::R12, RegState::Kill)
     546           2 :           .addReg(ARM::R4, RegState::Implicit)
     547             :           .setMIFlags(MachineInstr::FrameSetup);
     548           2 :       break;
     549             :     }
     550             : 
     551           9 :     BuildMI(MBB, MBBI, dl, TII.get(ARM::t2SUBrr), ARM::SP)
     552           3 :         .addReg(ARM::SP, RegState::Kill)
     553           3 :         .addReg(ARM::R4, RegState::Kill)
     554             :         .setMIFlags(MachineInstr::FrameSetup)
     555           3 :         .add(predOps(ARMCC::AL))
     556           3 :         .add(condCodeOp());
     557             :     NumBytes = 0;
     558             :   }
     559             : 
     560        3534 :   if (NumBytes) {
     561             :     // Adjust SP after all the callee-save spills.
     562        2183 :     if (AFI->getNumAlignedDPRCS2Regs() == 0 &&
     563        1076 :         tryFoldSPUpdateIntoPushPop(STI, MF, &*LastPush, NumBytes))
     564             :       DefCFAOffsetCandidates.addExtraBytes(LastPush, NumBytes);
     565             :     else {
     566        1086 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -NumBytes,
     567             :                    MachineInstr::FrameSetup);
     568             :       DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes);
     569             :     }
     570             : 
     571        1107 :     if (HasFP && isARM)
     572             :       // Restore from fp only in ARM mode: e.g. sub sp, r7, #24
     573             :       // Note it's not safe to do this in Thumb2 mode because it would have
     574             :       // taken two instructions:
     575             :       // mov sp, r7
     576             :       // sub sp, #24
     577             :       // If an interrupt is taken between the two instructions, then sp is in
     578             :       // an inconsistent state (pointing to the middle of callee-saved area).
     579             :       // The interrupt handler can end up clobbering the registers.
     580             :       AFI->setShouldRestoreSPFromFP(true);
     581             :   }
     582             : 
     583             :   // Set FP to point to the stack slot that contains the previous FP.
     584             :   // For iOS, FP is R7, which has now been stored in spill area 1.
     585             :   // Otherwise, if this is not iOS, all the callee-saved registers go
     586             :   // into spill area 1, including the FP in R11.  In either case, it
     587             :   // is in area one and the adjustment needs to take place just after
     588             :   // that push.
     589        3537 :   if (HasFP) {
     590         962 :     MachineBasicBlock::iterator AfterPush = std::next(GPRCS1Push);
     591         962 :     unsigned PushSize = sizeOfSPAdjustment(*GPRCS1Push);
     592         962 :     emitRegPlusImmediate(!AFI->isThumbFunction(), MBB, AfterPush,
     593             :                          dl, TII, FramePtr, ARM::SP,
     594         962 :                          PushSize + FramePtrOffsetInPush,
     595             :                          MachineInstr::FrameSetup);
     596         962 :     if (FramePtrOffsetInPush + PushSize != 0) {
     597        1428 :       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
     598         476 :           nullptr, MRI->getDwarfRegNum(FramePtr, true),
     599         476 :           -(ArgRegsSaveSize - FramePtrOffsetInPush)));
     600        1428 :       BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     601             :           .addCFIIndex(CFIIndex)
     602             :           .setMIFlags(MachineInstr::FrameSetup);
     603             :     } else {
     604             :       unsigned CFIIndex =
     605         972 :           MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
     606         486 :               nullptr, MRI->getDwarfRegNum(FramePtr, true)));
     607        1458 :       BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     608             :           .addCFIIndex(CFIIndex)
     609             :           .setMIFlags(MachineInstr::FrameSetup);
     610             :     }
     611             :   }
     612             : 
     613             :   // Now that the prologue's actual instructions are finalised, we can insert
     614             :   // the necessary DWARF cf instructions to describe the situation. Start by
     615             :   // recording where each register ended up:
     616        3537 :   if (GPRCS1Size > 0) {
     617        3530 :     MachineBasicBlock::iterator Pos = std::next(GPRCS1Push);
     618             :     int CFIIndex;
     619        3530 :     for (const auto &Entry : CSI) {
     620       11280 :       unsigned Reg = Entry.getReg();
     621       11280 :       int FI = Entry.getFrameIdx();
     622       11280 :       switch (Reg) {
     623        2112 :       case ARM::R8:
     624             :       case ARM::R9:
     625             :       case ARM::R10:
     626             :       case ARM::R11:
     627             :       case ARM::R12:
     628        2112 :         if (STI.splitFramePushPop(MF))
     629             :           break;
     630             :         LLVM_FALLTHROUGH;
     631             :       case ARM::R0:
     632             :       case ARM::R1:
     633             :       case ARM::R2:
     634             :       case ARM::R3:
     635             :       case ARM::R4:
     636             :       case ARM::R5:
     637             :       case ARM::R6:
     638             :       case ARM::R7:
     639             :       case ARM::LR:
     640       29430 :         CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
     641        9810 :             nullptr, MRI->getDwarfRegNum(Reg, true), MFI.getObjectOffset(FI)));
     642       29430 :         BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     643             :             .addCFIIndex(CFIIndex)
     644             :             .setMIFlags(MachineInstr::FrameSetup);
     645        9810 :         break;
     646             :       }
     647             :     }
     648             :   }
     649             : 
     650        3537 :   if (GPRCS2Size > 0) {
     651         116 :     MachineBasicBlock::iterator Pos = std::next(GPRCS2Push);
     652         116 :     for (const auto &Entry : CSI) {
     653        1113 :       unsigned Reg = Entry.getReg();
     654        1113 :       int FI = Entry.getFrameIdx();
     655        1113 :       switch (Reg) {
     656         212 :       case ARM::R8:
     657             :       case ARM::R9:
     658             :       case ARM::R10:
     659             :       case ARM::R11:
     660             :       case ARM::R12:
     661         212 :         if (STI.splitFramePushPop(MF)) {
     662         212 :           unsigned DwarfReg =  MRI->getDwarfRegNum(Reg, true);
     663             :           unsigned Offset = MFI.getObjectOffset(FI);
     664             :           unsigned CFIIndex = MF.addFrameInst(
     665         424 :               MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     666         636 :           BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     667             :               .addCFIIndex(CFIIndex)
     668             :               .setMIFlags(MachineInstr::FrameSetup);
     669             :         }
     670             :         break;
     671             :       }
     672             :     }
     673             :   }
     674             : 
     675        3537 :   if (DPRCSSize > 0) {
     676             :     // Since vpush register list cannot have gaps, there may be multiple vpush
     677             :     // instructions in the prologue.
     678         240 :     MachineBasicBlock::iterator Pos = std::next(LastPush);
     679         240 :     for (const auto &Entry : CSI) {
     680        1807 :       unsigned Reg = Entry.getReg();
     681        1807 :       int FI = Entry.getFrameIdx();
     682        1807 :       if ((Reg >= ARM::D0 && Reg <= ARM::D31) &&
     683         906 :           (Reg < ARM::D8 || Reg >= ARM::D8 + AFI->getNumAlignedDPRCS2Regs())) {
     684        1031 :         unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
     685             :         unsigned Offset = MFI.getObjectOffset(FI);
     686             :         unsigned CFIIndex = MF.addFrameInst(
     687        2062 :             MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     688        3093 :         BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     689             :             .addCFIIndex(CFIIndex)
     690             :             .setMIFlags(MachineInstr::FrameSetup);
     691             :       }
     692             :     }
     693             :   }
     694             : 
     695             :   // Now we can emit descriptions of where the canonical frame address was
     696             :   // throughout the process. If we have a frame pointer, it takes over the job
     697             :   // half-way through, so only the first few .cfi_def_cfa_offset instructions
     698             :   // actually get emitted.
     699        3537 :   DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
     700             : 
     701        7074 :   if (STI.isTargetELF() && hasFP(MF))
     702         742 :     MFI.setOffsetAdjustment(MFI.getOffsetAdjustment() -
     703         371 :                             AFI->getFramePtrSpillOffset());
     704             : 
     705             :   AFI->setGPRCalleeSavedArea1Size(GPRCS1Size);
     706             :   AFI->setGPRCalleeSavedArea2Size(GPRCS2Size);
     707             :   AFI->setDPRCalleeSavedGapSize(DPRGapSize);
     708             :   AFI->setDPRCalleeSavedAreaSize(DPRCSSize);
     709             : 
     710             :   // If we need dynamic stack realignment, do it here. Be paranoid and make
     711             :   // sure if we also have VLAs, we have a base pointer for frame access.
     712             :   // If aligned NEON registers were spilled, the stack has already been
     713             :   // realigned.
     714        3537 :   if (!AFI->getNumAlignedDPRCS2Regs() && RegInfo->needsStackRealignment(MF)) {
     715         164 :     unsigned MaxAlign = MFI.getMaxAlignment();
     716             :     assert(!AFI->isThumb1OnlyFunction());
     717         164 :     if (!AFI->isThumbFunction()) {
     718          98 :       emitAligningInstructions(MF, AFI, TII, MBB, MBBI, dl, ARM::SP, MaxAlign,
     719             :                                false);
     720             :     } else {
     721             :       // We cannot use sp as source/dest register here, thus we're using r4 to
     722             :       // perform the calculations. We're emitting the following sequence:
     723             :       // mov r4, sp
     724             :       // -- use emitAligningInstructions to produce best sequence to zero
     725             :       // -- out lower bits in r4
     726             :       // mov sp, r4
     727             :       // FIXME: It will be better just to find spare register here.
     728         198 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::R4)
     729          66 :           .addReg(ARM::SP, RegState::Kill)
     730          66 :           .add(predOps(ARMCC::AL));
     731          66 :       emitAligningInstructions(MF, AFI, TII, MBB, MBBI, dl, ARM::R4, MaxAlign,
     732             :                                false);
     733         198 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     734          66 :           .addReg(ARM::R4, RegState::Kill)
     735          66 :           .add(predOps(ARMCC::AL));
     736             :     }
     737             : 
     738             :     AFI->setShouldRestoreSPFromFP(true);
     739             :   }
     740             : 
     741             :   // If we need a base pointer, set it up here. It's whatever the value
     742             :   // of the stack pointer is at this point. Any variable size objects
     743             :   // will be allocated after this, so we can still use the base pointer
     744             :   // to reference locals.
     745             :   // FIXME: Clarify FrameSetup flags here.
     746        3537 :   if (RegInfo->hasBasePointer(MF)) {
     747           6 :     if (isARM)
     748           9 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), RegInfo->getBaseRegister())
     749           3 :           .addReg(ARM::SP)
     750           3 :           .add(predOps(ARMCC::AL))
     751           3 :           .add(condCodeOp());
     752             :     else
     753           9 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), RegInfo->getBaseRegister())
     754           3 :           .addReg(ARM::SP)
     755           3 :           .add(predOps(ARMCC::AL));
     756             :   }
     757             : 
     758             :   // If the frame has variable sized objects then the epilogue must restore
     759             :   // the sp from fp. We can assume there's an FP here since hasFP already
     760             :   // checks for hasVarSizedObjects.
     761        3537 :   if (MFI.hasVarSizedObjects())
     762             :     AFI->setShouldRestoreSPFromFP(true);
     763             : }
     764             : 
     765       12556 : void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
     766             :                                     MachineBasicBlock &MBB) const {
     767       12556 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     768       12556 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     769       12556 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     770             :   const ARMBaseInstrInfo &TII =
     771       12556 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
     772             :   assert(!AFI->isThumb1OnlyFunction() &&
     773             :          "This emitEpilogue does not support Thumb1!");
     774       12556 :   bool isARM = !AFI->isThumbFunction();
     775             : 
     776       12556 :   unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
     777       12556 :   int NumBytes = (int)MFI.getStackSize();
     778       12556 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
     779             : 
     780             :   // All calls are tail calls in GHC calling conv, and functions have no
     781             :   // prologue/epilogue.
     782       25112 :   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     783           2 :     return;
     784             : 
     785             :   // First put ourselves on the first (from top) terminator instructions.
     786       12554 :   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
     787       12580 :   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
     788             : 
     789       12554 :   if (!AFI->hasStackFrame()) {
     790        9064 :     if (NumBytes - ArgRegsSaveSize != 0)
     791         386 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ArgRegsSaveSize);
     792             :   } else {
     793             :     // Unwind MBBI to point to first LDR / VLDRD.
     794        3490 :     const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
     795        3490 :     if (MBBI != MBB.begin()) {
     796             :       do {
     797             :         --MBBI;
     798        4777 :       } while (MBBI != MBB.begin() && isCSRestore(*MBBI, TII, CSRegs));
     799        3354 :       if (!isCSRestore(*MBBI, TII, CSRegs))
     800             :         ++MBBI;
     801             :     }
     802             : 
     803             :     // Move SP to start of FP callee save spill area.
     804        6980 :     NumBytes -= (ArgRegsSaveSize +
     805        6980 :                  AFI->getGPRCalleeSavedArea1Size() +
     806        6980 :                  AFI->getGPRCalleeSavedArea2Size() +
     807        6980 :                  AFI->getDPRCalleeSavedGapSize() +
     808        3490 :                  AFI->getDPRCalleeSavedAreaSize());
     809             : 
     810             :     // Reset SP based on frame pointer only if the stack frame extends beyond
     811             :     // frame pointer stack slot or target is ELF and the function has FP.
     812        3490 :     if (AFI->shouldRestoreSPFromFP()) {
     813         379 :       NumBytes = AFI->getFramePtrSpillOffset() - NumBytes;
     814         379 :       if (NumBytes) {
     815         251 :         if (isARM)
     816         151 :           emitARMRegPlusImmediate(MBB, MBBI, dl, ARM::SP, FramePtr, -NumBytes,
     817             :                                   ARMCC::AL, 0, TII);
     818             :         else {
     819             :           // It's not possible to restore SP from FP in a single instruction.
     820             :           // For iOS, this looks like:
     821             :           // mov sp, r7
     822             :           // sub sp, #24
     823             :           // This is bad, if an interrupt is taken after the mov, sp is in an
     824             :           // inconsistent state.
     825             :           // Use the first callee-saved register as a scratch register.
     826             :           assert(!MFI.getPristineRegs(MF).test(ARM::R4) &&
     827             :                  "No scratch register to restore SP from FP!");
     828         100 :           emitT2RegPlusImmediate(MBB, MBBI, dl, ARM::R4, FramePtr, -NumBytes,
     829             :                                  ARMCC::AL, 0, TII);
     830         300 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     831         100 :               .addReg(ARM::R4)
     832         100 :               .add(predOps(ARMCC::AL));
     833             :         }
     834             :       } else {
     835             :         // Thumb2 or ARM.
     836         128 :         if (isARM)
     837         384 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), ARM::SP)
     838         128 :               .addReg(FramePtr)
     839         128 :               .add(predOps(ARMCC::AL))
     840         128 :               .add(condCodeOp());
     841             :         else
     842           0 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     843           0 :               .addReg(FramePtr)
     844           0 :               .add(predOps(ARMCC::AL));
     845             :       }
     846        3841 :     } else if (NumBytes &&
     847         730 :                !tryFoldSPUpdateIntoPushPop(STI, MF, &*MBBI, NumBytes))
     848         717 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes);
     849             : 
     850             :     // Increment past our save areas.
     851        3490 :     if (MBBI != MBB.end() && AFI->getDPRCalleeSavedAreaSize()) {
     852             :       MBBI++;
     853             :       // Since vpop register list cannot have gaps, there may be multiple vpop
     854             :       // instructions in the epilogue.
     855         550 :       while (MBBI != MBB.end() && MBBI->getOpcode() == ARM::VLDMDIA_UPD)
     856             :         MBBI++;
     857             :     }
     858        3490 :     if (AFI->getDPRCalleeSavedGapSize()) {
     859             :       assert(AFI->getDPRCalleeSavedGapSize() == 4 &&
     860             :              "unexpected DPR alignment gap");
     861          13 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, AFI->getDPRCalleeSavedGapSize());
     862             :     }
     863             : 
     864        3490 :     if (AFI->getGPRCalleeSavedArea2Size()) MBBI++;
     865        3490 :     if (AFI->getGPRCalleeSavedArea1Size()) MBBI++;
     866             :   }
     867             : 
     868       12554 :   if (ArgRegsSaveSize)
     869          44 :     emitSPUpdate(isARM, MBB, MBBI, dl, TII, ArgRegsSaveSize);
     870             : }
     871             : 
     872             : /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
     873             : /// debug info.  It's the same as what we use for resolving the code-gen
     874             : /// references for now.  FIXME: This can go wrong when references are
     875             : /// SP-relative and simple call frames aren't used.
     876             : int
     877          22 : ARMFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
     878             :                                          unsigned &FrameReg) const {
     879          22 :   return ResolveFrameIndexReference(MF, FI, FrameReg, 0);
     880             : }
     881             : 
     882             : int
     883       14961 : ARMFrameLowering::ResolveFrameIndexReference(const MachineFunction &MF,
     884             :                                              int FI, unsigned &FrameReg,
     885             :                                              int SPAdj) const {
     886       14961 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     887             :   const ARMBaseRegisterInfo *RegInfo = static_cast<const ARMBaseRegisterInfo *>(
     888       14961 :       MF.getSubtarget().getRegisterInfo());
     889             :   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     890       14961 :   int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
     891       14961 :   int FPOffset = Offset - AFI->getFramePtrSpillOffset();
     892             :   bool isFixed = MFI.isFixedObjectIndex(FI);
     893             : 
     894       14961 :   FrameReg = ARM::SP;
     895       14961 :   Offset += SPAdj;
     896             : 
     897             :   // SP can move around if there are allocas.  We may also lose track of SP
     898             :   // when emergency spilling inside a non-reserved call frame setup.
     899       14961 :   bool hasMovingSP = !hasReservedCallFrame(MF);
     900             : 
     901             :   // When dynamically realigning the stack, use the frame pointer for
     902             :   // parameters, and the stack/base pointer for locals.
     903       14961 :   if (RegInfo->needsStackRealignment(MF)) {
     904             :     assert(hasFP(MF) && "dynamic stack realignment without a FP!");
     905        7225 :     if (isFixed) {
     906          66 :       FrameReg = RegInfo->getFrameRegister(MF);
     907             :       Offset = FPOffset;
     908        7159 :     } else if (hasMovingSP) {
     909             :       assert(RegInfo->hasBasePointer(MF) &&
     910             :              "VLAs and dynamic stack alignment, but missing base pointer!");
     911         468 :       FrameReg = RegInfo->getBaseRegister();
     912             :     }
     913             :     return Offset;
     914             :   }
     915             : 
     916             :   // If there is a frame pointer, use it when we can.
     917        7736 :   if (hasFP(MF) && AFI->hasStackFrame()) {
     918             :     // Use frame pointer to reference fixed objects. Use it for locals if
     919             :     // there are VLAs (and thus the SP isn't reliable as a base).
     920        1682 :     if (isFixed || (hasMovingSP && !RegInfo->hasBasePointer(MF))) {
     921         273 :       FrameReg = RegInfo->getFrameRegister(MF);
     922         273 :       return FPOffset;
     923        1409 :     } else if (hasMovingSP) {
     924             :       assert(RegInfo->hasBasePointer(MF) && "missing base pointer!");
     925          35 :       if (AFI->isThumb2Function()) {
     926             :         // Try to use the frame pointer if we can, else use the base pointer
     927             :         // since it's available. This is handy for the emergency spill slot, in
     928             :         // particular.
     929           0 :         if (FPOffset >= -255 && FPOffset < 0) {
     930           0 :           FrameReg = RegInfo->getFrameRegister(MF);
     931           0 :           return FPOffset;
     932             :         }
     933             :       }
     934        1374 :     } else if (AFI->isThumbFunction()) {
     935             :       // Prefer SP to base pointer, if the offset is suitably aligned and in
     936             :       // range as the effective range of the immediate offset is bigger when
     937             :       // basing off SP.
     938             :       // Use  add <rd>, sp, #<imm8>
     939             :       //      ldr <rd>, [sp, #<imm8>]
     940         518 :       if (Offset >= 0 && (Offset & 3) == 0 && Offset <= 1020)
     941             :         return Offset;
     942             :       // In Thumb2 mode, the negative offset is very limited. Try to avoid
     943             :       // out of range references. ldr <rt>,[<rn>, #-<imm8>]
     944          24 :       if (AFI->isThumb2Function() && FPOffset >= -255 && FPOffset < 0) {
     945          24 :         FrameReg = RegInfo->getFrameRegister(MF);
     946          24 :         return FPOffset;
     947             :       }
     948         856 :     } else if (Offset > (FPOffset < 0 ? -FPOffset : FPOffset)) {
     949             :       // Otherwise, use SP or FP, whichever is closer to the stack slot.
     950         253 :       FrameReg = RegInfo->getFrameRegister(MF);
     951         253 :       return FPOffset;
     952             :     }
     953             :   }
     954             :   // Use the base pointer if we have one.
     955        6694 :   if (RegInfo->hasBasePointer(MF))
     956          35 :     FrameReg = RegInfo->getBaseRegister();
     957             :   return Offset;
     958             : }
     959             : 
     960       10611 : void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
     961             :                                     MachineBasicBlock::iterator MI,
     962             :                                     const std::vector<CalleeSavedInfo> &CSI,
     963             :                                     unsigned StmOpc, unsigned StrOpc,
     964             :                                     bool NoGap,
     965             :                                     bool(*Func)(unsigned, bool),
     966             :                                     unsigned NumAlignedDPRCS2Regs,
     967             :                                     unsigned MIFlags) const {
     968       10611 :   MachineFunction &MF = *MBB.getParent();
     969       10611 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
     970       10611 :   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
     971             : 
     972       10611 :   DebugLoc DL;
     973             : 
     974             :   using RegAndKill = std::pair<unsigned, bool>;
     975             : 
     976             :   SmallVector<RegAndKill, 4> Regs;
     977       21222 :   unsigned i = CSI.size();
     978       21248 :   while (i != 0) {
     979             :     unsigned LastReg = 0;
     980       44612 :     for (; i != 0; --i) {
     981       68002 :       unsigned Reg = CSI[i-1].getReg();
     982       34001 :       if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue;
     983             : 
     984             :       // D-registers in the aligned area DPRCS2 are NOT spilled here.
     985       11351 :       if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs)
     986         272 :         continue;
     987             : 
     988       11079 :       const MachineRegisterInfo &MRI = MF.getRegInfo();
     989       11079 :       bool isLiveIn = MRI.isLiveIn(Reg);
     990       22137 :       if (!isLiveIn && !MRI.isReserved(Reg))
     991       10090 :         MBB.addLiveIn(Reg);
     992             :       // If NoGap is true, push consecutive registers and then leave the rest
     993             :       // for other instructions. e.g.
     994             :       // vpush {d8, d10, d11} -> vpush {d8}, vpush {d10, d11}
     995       11079 :       if (NoGap && LastReg && LastReg != Reg-1)
     996             :         break;
     997             :       LastReg = Reg;
     998             :       // Do not set a kill flag on values that are also marked as live-in. This
     999             :       // happens with the @llvm-returnaddress intrinsic and with arguments
    1000             :       // passed in callee saved registers.
    1001             :       // Omitting the kill flags is conservatively correct even if the live-in
    1002             :       // is not used after all.
    1003       11053 :       Regs.push_back(std::make_pair(Reg, /*isKill=*/!isLiveIn));
    1004             :     }
    1005             : 
    1006       10637 :     if (Regs.empty())
    1007        6725 :       continue;
    1008             : 
    1009             :     llvm::sort(Regs.begin(), Regs.end(), [&](const RegAndKill &LHS,
    1010             :                                              const RegAndKill &RHS) {
    1011       14554 :       return TRI.getEncodingValue(LHS.first) < TRI.getEncodingValue(RHS.first);
    1012             :     });
    1013             : 
    1014        3912 :     if (Regs.size() > 1 || StrOpc== 0) {
    1015       10911 :       MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StmOpc), ARM::SP)
    1016        3637 :                                     .addReg(ARM::SP)
    1017             :                                     .setMIFlags(MIFlags)
    1018        3637 :                                     .add(predOps(ARMCC::AL));
    1019       14415 :       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
    1020       32334 :         MIB.addReg(Regs[i].first, getKillRegState(Regs[i].second));
    1021         275 :     } else if (Regs.size() == 1) {
    1022         825 :       BuildMI(MBB, MI, DL, TII.get(StrOpc), ARM::SP)
    1023         550 :           .addReg(Regs[0].first, getKillRegState(Regs[0].second))
    1024         275 :           .addReg(ARM::SP)
    1025             :           .setMIFlags(MIFlags)
    1026             :           .addImm(-4)
    1027         275 :           .add(predOps(ARMCC::AL));
    1028             :     }
    1029             :     Regs.clear();
    1030             : 
    1031             :     // Put any subsequent vpush instructions before this one: they will refer to
    1032             :     // higher register numbers so need to be pushed first in order to preserve
    1033             :     // monotonicity.
    1034        3912 :     if (MI != MBB.begin())
    1035             :       --MI;
    1036             :   }
    1037       10611 : }
    1038             : 
    1039       10470 : void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
    1040             :                                    MachineBasicBlock::iterator MI,
    1041             :                                    std::vector<CalleeSavedInfo> &CSI,
    1042             :                                    unsigned LdmOpc, unsigned LdrOpc,
    1043             :                                    bool isVarArg, bool NoGap,
    1044             :                                    bool(*Func)(unsigned, bool),
    1045             :                                    unsigned NumAlignedDPRCS2Regs) const {
    1046       10470 :   MachineFunction &MF = *MBB.getParent();
    1047       10470 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1048       10470 :   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
    1049       10470 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1050       10470 :   DebugLoc DL;
    1051             :   bool isTailCall = false;
    1052             :   bool isInterrupt = false;
    1053             :   bool isTrap = false;
    1054       10470 :   if (MBB.end() != MI) {
    1055             :     DL = MI->getDebugLoc();
    1056       10395 :     unsigned RetOpcode = MI->getOpcode();
    1057       10395 :     isTailCall = (RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNri);
    1058       10395 :     isInterrupt =
    1059       10395 :         RetOpcode == ARM::SUBS_PC_LR || RetOpcode == ARM::t2SUBS_PC_LR;
    1060       10395 :     isTrap =
    1061       10395 :         RetOpcode == ARM::TRAP || RetOpcode == ARM::TRAPNaCl ||
    1062       10395 :         RetOpcode == ARM::tTRAP;
    1063             :   }
    1064             : 
    1065             :   SmallVector<unsigned, 4> Regs;
    1066       20940 :   unsigned i = CSI.size();
    1067       20964 :   while (i != 0) {
    1068             :     unsigned LastReg = 0;
    1069             :     bool DeleteRet = false;
    1070       43821 :     for (; i != 0; --i) {
    1071       33351 :       CalleeSavedInfo &Info = CSI[i-1];
    1072       33351 :       unsigned Reg = Info.getReg();
    1073       55777 :       if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue;
    1074             : 
    1075             :       // The aligned reloads from area DPRCS2 are not inserted here.
    1076       11133 :       if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs)
    1077         208 :         continue;
    1078             : 
    1079        3456 :       if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt &&
    1080       14255 :           !isTrap && STI.hasV5TOps()) {
    1081        2545 :         if (MBB.succ_empty()) {
    1082        2509 :           Reg = ARM::PC;
    1083             :           // Fold the return instruction into the LDM.
    1084             :           DeleteRet = true;
    1085        2509 :           LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_RET : ARM::LDMIA_RET;
    1086             :           // We 'restore' LR into PC so it is not live out of the return block:
    1087             :           // Clear Restored bit.
    1088             :           Info.setRestored(false);
    1089             :         } else
    1090          36 :           LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
    1091             :       }
    1092             : 
    1093             :       // If NoGap is true, pop consecutive registers and then leave the rest
    1094             :       // for other instructions. e.g.
    1095             :       // vpop {d8, d10, d11} -> vpop {d8}, vpop {d10, d11}
    1096       10925 :       if (NoGap && LastReg && LastReg != Reg-1)
    1097             :         break;
    1098             : 
    1099       10901 :       LastReg = Reg;
    1100       10901 :       Regs.push_back(Reg);
    1101             :     }
    1102             : 
    1103       10494 :     if (Regs.empty())
    1104        6621 :       continue;
    1105             : 
    1106             :     llvm::sort(Regs.begin(), Regs.end(), [&](unsigned LHS, unsigned RHS) {
    1107        7028 :       return TRI.getEncodingValue(LHS) < TRI.getEncodingValue(RHS);
    1108             :     });
    1109             : 
    1110        3873 :     if (Regs.size() > 1 || LdrOpc == 0) {
    1111       10821 :       MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdmOpc), ARM::SP)
    1112        3607 :                                     .addReg(ARM::SP)
    1113        3607 :                                     .add(predOps(ARMCC::AL));
    1114       14242 :       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
    1115       21270 :         MIB.addReg(Regs[i], getDefRegState(true));
    1116        3607 :       if (DeleteRet) {
    1117        2367 :         if (MI != MBB.end()) {
    1118             :           MIB.copyImplicitOps(*MI);
    1119        2367 :           MI->eraseFromParent();
    1120             :         }
    1121             :       }
    1122        3607 :       MI = MIB;
    1123         266 :     } else if (Regs.size() == 1) {
    1124             :       // If we adjusted the reg to PC from LR above, switch it back here. We
    1125             :       // only do that for LDM.
    1126         266 :       if (Regs[0] == ARM::PC)
    1127         142 :         Regs[0] = ARM::LR;
    1128             :       MachineInstrBuilder MIB =
    1129         798 :         BuildMI(MBB, MI, DL, TII.get(LdrOpc), Regs[0])
    1130         266 :           .addReg(ARM::SP, RegState::Define)
    1131         266 :           .addReg(ARM::SP);
    1132             :       // ARM mode needs an extra reg0 here due to addrmode2. Will go away once
    1133             :       // that refactoring is complete (eventually).
    1134         266 :       if (LdrOpc == ARM::LDR_POST_REG || LdrOpc == ARM::LDR_POST_IMM) {
    1135         192 :         MIB.addReg(0);
    1136             :         MIB.addImm(ARM_AM::getAM2Opc(ARM_AM::add, 4, ARM_AM::no_shift));
    1137             :       } else
    1138             :         MIB.addImm(4);
    1139         266 :       MIB.add(predOps(ARMCC::AL));
    1140             :     }
    1141             :     Regs.clear();
    1142             : 
    1143             :     // Put any subsequent vpop instructions after this one: they will refer to
    1144             :     // higher register numbers so need to be popped afterwards.
    1145        3873 :     if (MI != MBB.end())
    1146             :       ++MI;
    1147             :   }
    1148       10470 : }
    1149             : 
    1150             : /// Emit aligned spill instructions for NumAlignedDPRCS2Regs D-registers
    1151             : /// starting from d8.  Also insert stack realignment code and leave the stack
    1152             : /// pointer pointing to the d8 spill slot.
    1153          40 : static void emitAlignedDPRCS2Spills(MachineBasicBlock &MBB,
    1154             :                                     MachineBasicBlock::iterator MI,
    1155             :                                     unsigned NumAlignedDPRCS2Regs,
    1156             :                                     const std::vector<CalleeSavedInfo> &CSI,
    1157             :                                     const TargetRegisterInfo *TRI) {
    1158          40 :   MachineFunction &MF = *MBB.getParent();
    1159          40 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1160          40 :   DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
    1161          40 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1162          40 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1163             : 
    1164             :   // Mark the D-register spill slots as properly aligned.  Since MFI computes
    1165             :   // stack slot layout backwards, this can actually mean that the d-reg stack
    1166             :   // slot offsets can be wrong. The offset for d8 will always be correct.
    1167         604 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
    1168        1048 :     unsigned DNum = CSI[i].getReg() - ARM::D8;
    1169         524 :     if (DNum > NumAlignedDPRCS2Regs - 1)
    1170         252 :       continue;
    1171         272 :     int FI = CSI[i].getFrameIdx();
    1172             :     // The even-numbered registers will be 16-byte aligned, the odd-numbered
    1173             :     // registers will be 8-byte aligned.
    1174         272 :     MFI.setObjectAlignment(FI, DNum % 2 ? 8 : 16);
    1175             : 
    1176             :     // The stack slot for D8 needs to be maximally aligned because this is
    1177             :     // actually the point where we align the stack pointer.  MachineFrameInfo
    1178             :     // computes all offsets relative to the incoming stack pointer which is a
    1179             :     // bit weird when realigning the stack.  Any extra padding for this
    1180             :     // over-alignment is not realized because the code inserted below adjusts
    1181             :     // the stack pointer by numregs * 8 before aligning the stack pointer.
    1182         272 :     if (DNum == 0)
    1183          40 :       MFI.setObjectAlignment(FI, MFI.getMaxAlignment());
    1184             :   }
    1185             : 
    1186             :   // Move the stack pointer to the d8 spill slot, and align it at the same
    1187             :   // time. Leave the stack slot address in the scratch register r4.
    1188             :   //
    1189             :   //   sub r4, sp, #numregs * 8
    1190             :   //   bic r4, r4, #align - 1
    1191             :   //   mov sp, r4
    1192             :   //
    1193          40 :   bool isThumb = AFI->isThumbFunction();
    1194             :   assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
    1195             :   AFI->setShouldRestoreSPFromFP(true);
    1196             : 
    1197             :   // sub r4, sp, #numregs * 8
    1198             :   // The immediate is <= 64, so it doesn't need any special encoding.
    1199          40 :   unsigned Opc = isThumb ? ARM::t2SUBri : ARM::SUBri;
    1200         120 :   BuildMI(MBB, MI, DL, TII.get(Opc), ARM::R4)
    1201          40 :       .addReg(ARM::SP)
    1202          40 :       .addImm(8 * NumAlignedDPRCS2Regs)
    1203          40 :       .add(predOps(ARMCC::AL))
    1204          40 :       .add(condCodeOp());
    1205             : 
    1206          40 :   unsigned MaxAlign = MF.getFrameInfo().getMaxAlignment();
    1207             :   // We must set parameter MustBeSingleInstruction to true, since
    1208             :   // skipAlignedDPRCS2Spills expects exactly 3 instructions to perform
    1209             :   // stack alignment.  Luckily, this can always be done since all ARM
    1210             :   // architecture versions that support Neon also support the BFC
    1211             :   // instruction.
    1212          40 :   emitAligningInstructions(MF, AFI, TII, MBB, MI, DL, ARM::R4, MaxAlign, true);
    1213             : 
    1214             :   // mov sp, r4
    1215             :   // The stack pointer must be adjusted before spilling anything, otherwise
    1216             :   // the stack slots could be clobbered by an interrupt handler.
    1217             :   // Leave r4 live, it is used below.
    1218          40 :   Opc = isThumb ? ARM::tMOVr : ARM::MOVr;
    1219         120 :   MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(Opc), ARM::SP)
    1220          40 :                                 .addReg(ARM::R4)
    1221          40 :                                 .add(predOps(ARMCC::AL));
    1222          40 :   if (!isThumb)
    1223          16 :     MIB.add(condCodeOp());
    1224             : 
    1225             :   // Now spill NumAlignedDPRCS2Regs registers starting from d8.
    1226             :   // r4 holds the stack slot address.
    1227             :   unsigned NextReg = ARM::D8;
    1228             : 
    1229             :   // 16-byte aligned vst1.64 with 4 d-regs and address writeback.
    1230             :   // The writeback is only needed when emitting two vst1.64 instructions.
    1231          40 :   if (NumAlignedDPRCS2Regs >= 6) {
    1232             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1233          32 :                                                &ARM::QQPRRegClass);
    1234          32 :     MBB.addLiveIn(SupReg);
    1235          96 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1d64Qwb_fixed), ARM::R4)
    1236          32 :         .addReg(ARM::R4, RegState::Kill)
    1237             :         .addImm(16)
    1238          32 :         .addReg(NextReg)
    1239          32 :         .addReg(SupReg, RegState::ImplicitKill)
    1240          32 :         .add(predOps(ARMCC::AL));
    1241             :     NextReg += 4;
    1242          32 :     NumAlignedDPRCS2Regs -= 4;
    1243             :   }
    1244             : 
    1245             :   // We won't modify r4 beyond this point.  It currently points to the next
    1246             :   // register to be spilled.
    1247             :   unsigned R4BaseReg = NextReg;
    1248             : 
    1249             :   // 16-byte aligned vst1.64 with 4 d-regs, no writeback.
    1250          40 :   if (NumAlignedDPRCS2Regs >= 4) {
    1251             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1252          31 :                                                &ARM::QQPRRegClass);
    1253          31 :     MBB.addLiveIn(SupReg);
    1254          93 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1d64Q))
    1255          31 :         .addReg(ARM::R4)
    1256             :         .addImm(16)
    1257          31 :         .addReg(NextReg)
    1258          31 :         .addReg(SupReg, RegState::ImplicitKill)
    1259          31 :         .add(predOps(ARMCC::AL));
    1260          31 :     NextReg += 4;
    1261          31 :     NumAlignedDPRCS2Regs -= 4;
    1262             :   }
    1263             : 
    1264             :   // 16-byte aligned vst1.64 with 2 d-regs.
    1265          40 :   if (NumAlignedDPRCS2Regs >= 2) {
    1266             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1267           9 :                                                &ARM::QPRRegClass);
    1268           9 :     MBB.addLiveIn(SupReg);
    1269          27 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1q64))
    1270           9 :         .addReg(ARM::R4)
    1271             :         .addImm(16)
    1272           9 :         .addReg(SupReg)
    1273           9 :         .add(predOps(ARMCC::AL));
    1274           9 :     NextReg += 2;
    1275           9 :     NumAlignedDPRCS2Regs -= 2;
    1276             :   }
    1277             : 
    1278             :   // Finally, use a vanilla vstr.64 for the odd last register.
    1279          40 :   if (NumAlignedDPRCS2Regs) {
    1280           2 :     MBB.addLiveIn(NextReg);
    1281             :     // vstr.64 uses addrmode5 which has an offset scale of 4.
    1282           6 :     BuildMI(MBB, MI, DL, TII.get(ARM::VSTRD))
    1283           2 :         .addReg(NextReg)
    1284           2 :         .addReg(ARM::R4)
    1285           2 :         .addImm((NextReg - R4BaseReg) * 2)
    1286           2 :         .add(predOps(ARMCC::AL));
    1287             :   }
    1288             : 
    1289             :   // The last spill instruction inserted should kill the scratch register r4.
    1290          40 :   std::prev(MI)->addRegisterKilled(ARM::R4, TRI);
    1291          40 : }
    1292             : 
    1293             : /// Skip past the code inserted by emitAlignedDPRCS2Spills, and return an
    1294             : /// iterator to the following instruction.
    1295             : static MachineBasicBlock::iterator
    1296          40 : skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
    1297             :                         unsigned NumAlignedDPRCS2Regs) {
    1298             :   //   sub r4, sp, #numregs * 8
    1299             :   //   bic r4, r4, #align - 1
    1300             :   //   mov sp, r4
    1301             :   ++MI; ++MI; ++MI;
    1302             :   assert(MI->mayStore() && "Expecting spill instruction");
    1303             : 
    1304             :   // These switches all fall through.
    1305          40 :   switch(NumAlignedDPRCS2Regs) {
    1306             :   case 7:
    1307             :     ++MI;
    1308             :     assert(MI->mayStore() && "Expecting spill instruction");
    1309             :     LLVM_FALLTHROUGH;
    1310          33 :   default:
    1311             :     ++MI;
    1312             :     assert(MI->mayStore() && "Expecting spill instruction");
    1313             :     LLVM_FALLTHROUGH;
    1314          40 :   case 1:
    1315             :   case 2:
    1316             :   case 4:
    1317             :     assert(MI->killsRegister(ARM::R4) && "Missed kill flag");
    1318             :     ++MI;
    1319             :   }
    1320          40 :   return MI;
    1321             : }
    1322             : 
    1323             : /// Emit aligned reload instructions for NumAlignedDPRCS2Regs D-registers
    1324             : /// starting from d8.  These instructions are assumed to execute while the
    1325             : /// stack is still aligned, unlike the code inserted by emitPopInst.
    1326          32 : static void emitAlignedDPRCS2Restores(MachineBasicBlock &MBB,
    1327             :                                       MachineBasicBlock::iterator MI,
    1328             :                                       unsigned NumAlignedDPRCS2Regs,
    1329             :                                       const std::vector<CalleeSavedInfo> &CSI,
    1330             :                                       const TargetRegisterInfo *TRI) {
    1331          32 :   MachineFunction &MF = *MBB.getParent();
    1332          32 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1333          34 :   DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
    1334          32 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1335             : 
    1336             :   // Find the frame index assigned to d8.
    1337             :   int D8SpillFI = 0;
    1338         428 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i)
    1339         792 :     if (CSI[i].getReg() == ARM::D8) {
    1340          32 :       D8SpillFI = CSI[i].getFrameIdx();
    1341          32 :       break;
    1342             :     }
    1343             : 
    1344             :   // Materialize the address of the d8 spill slot into the scratch register r4.
    1345             :   // This can be fairly complicated if the stack frame is large, so just use
    1346             :   // the normal frame index elimination mechanism to do it.  This code runs as
    1347             :   // the initial part of the epilog where the stack and base pointers haven't
    1348             :   // been changed yet.
    1349          32 :   bool isThumb = AFI->isThumbFunction();
    1350             :   assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
    1351             : 
    1352          32 :   unsigned Opc = isThumb ? ARM::t2ADDri : ARM::ADDri;
    1353          64 :   BuildMI(MBB, MI, DL, TII.get(Opc), ARM::R4)
    1354             :       .addFrameIndex(D8SpillFI)
    1355             :       .addImm(0)
    1356          32 :       .add(predOps(ARMCC::AL))
    1357          32 :       .add(condCodeOp());
    1358             : 
    1359             :   // Now restore NumAlignedDPRCS2Regs registers starting from d8.
    1360             :   unsigned NextReg = ARM::D8;
    1361             : 
    1362             :   // 16-byte aligned vld1.64 with 4 d-regs and writeback.
    1363          32 :   if (NumAlignedDPRCS2Regs >= 6) {
    1364             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1365          24 :                                                &ARM::QQPRRegClass);
    1366          72 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1d64Qwb_fixed), NextReg)
    1367          24 :         .addReg(ARM::R4, RegState::Define)
    1368          24 :         .addReg(ARM::R4, RegState::Kill)
    1369             :         .addImm(16)
    1370          24 :         .addReg(SupReg, RegState::ImplicitDefine)
    1371          24 :         .add(predOps(ARMCC::AL));
    1372             :     NextReg += 4;
    1373          24 :     NumAlignedDPRCS2Regs -= 4;
    1374             :   }
    1375             : 
    1376             :   // We won't modify r4 beyond this point.  It currently points to the next
    1377             :   // register to be spilled.
    1378             :   unsigned R4BaseReg = NextReg;
    1379             : 
    1380             :   // 16-byte aligned vld1.64 with 4 d-regs, no writeback.
    1381          32 :   if (NumAlignedDPRCS2Regs >= 4) {
    1382             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1383          23 :                                                &ARM::QQPRRegClass);
    1384          69 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1d64Q), NextReg)
    1385          23 :         .addReg(ARM::R4)
    1386             :         .addImm(16)
    1387          23 :         .addReg(SupReg, RegState::ImplicitDefine)
    1388          23 :         .add(predOps(ARMCC::AL));
    1389          23 :     NextReg += 4;
    1390          23 :     NumAlignedDPRCS2Regs -= 4;
    1391             :   }
    1392             : 
    1393             :   // 16-byte aligned vld1.64 with 2 d-regs.
    1394          32 :   if (NumAlignedDPRCS2Regs >= 2) {
    1395             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1396           9 :                                                &ARM::QPRRegClass);
    1397          27 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1q64), SupReg)
    1398           9 :         .addReg(ARM::R4)
    1399             :         .addImm(16)
    1400           9 :         .add(predOps(ARMCC::AL));
    1401           9 :     NextReg += 2;
    1402           9 :     NumAlignedDPRCS2Regs -= 2;
    1403             :   }
    1404             : 
    1405             :   // Finally, use a vanilla vldr.64 for the remaining odd register.
    1406          32 :   if (NumAlignedDPRCS2Regs)
    1407           6 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLDRD), NextReg)
    1408           2 :         .addReg(ARM::R4)
    1409           2 :         .addImm(2 * (NextReg - R4BaseReg))
    1410           2 :         .add(predOps(ARMCC::AL));
    1411             : 
    1412             :   // Last store kills r4.
    1413          32 :   std::prev(MI)->addRegisterKilled(ARM::R4, TRI);
    1414          32 : }
    1415             : 
    1416        3537 : bool ARMFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
    1417             :                                         MachineBasicBlock::iterator MI,
    1418             :                                         const std::vector<CalleeSavedInfo> &CSI,
    1419             :                                         const TargetRegisterInfo *TRI) const {
    1420        3537 :   if (CSI.empty())
    1421             :     return false;
    1422             : 
    1423        3537 :   MachineFunction &MF = *MBB.getParent();
    1424        3537 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1425             : 
    1426        3537 :   unsigned PushOpc = AFI->isThumbFunction() ? ARM::t2STMDB_UPD : ARM::STMDB_UPD;
    1427        3537 :   unsigned PushOneOpc = AFI->isThumbFunction() ?
    1428             :     ARM::t2STR_PRE : ARM::STR_PRE_IMM;
    1429             :   unsigned FltOpc = ARM::VSTMDDB_UPD;
    1430        3537 :   unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
    1431        3537 :   emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isARMArea1Register, 0,
    1432             :                MachineInstr::FrameSetup);
    1433        3537 :   emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isARMArea2Register, 0,
    1434             :                MachineInstr::FrameSetup);
    1435        3537 :   emitPushInst(MBB, MI, CSI, FltOpc, 0, true, &isARMArea3Register,
    1436             :                NumAlignedDPRCS2Regs, MachineInstr::FrameSetup);
    1437             : 
    1438             :   // The code above does not insert spill code for the aligned DPRCS2 registers.
    1439             :   // The stack realignment code will be inserted between the push instructions
    1440             :   // and these spills.
    1441        3537 :   if (NumAlignedDPRCS2Regs)
    1442          40 :     emitAlignedDPRCS2Spills(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
    1443             : 
    1444             :   return true;
    1445             : }
    1446             : 
    1447        3490 : bool ARMFrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
    1448             :                                         MachineBasicBlock::iterator MI,
    1449             :                                         std::vector<CalleeSavedInfo> &CSI,
    1450             :                                         const TargetRegisterInfo *TRI) const {
    1451        3490 :   if (CSI.empty())
    1452             :     return false;
    1453             : 
    1454        3490 :   MachineFunction &MF = *MBB.getParent();
    1455        3490 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1456        3490 :   bool isVarArg = AFI->getArgRegsSaveSize() > 0;
    1457        3490 :   unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
    1458             : 
    1459             :   // The emitPopInst calls below do not insert reloads for the aligned DPRCS2
    1460             :   // registers. Do that here instead.
    1461        3490 :   if (NumAlignedDPRCS2Regs)
    1462          32 :     emitAlignedDPRCS2Restores(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
    1463             : 
    1464        3490 :   unsigned PopOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
    1465        3490 :   unsigned LdrOpc = AFI->isThumbFunction() ? ARM::t2LDR_POST :ARM::LDR_POST_IMM;
    1466             :   unsigned FltOpc = ARM::VLDMDIA_UPD;
    1467        3490 :   emitPopInst(MBB, MI, CSI, FltOpc, 0, isVarArg, true, &isARMArea3Register,
    1468             :               NumAlignedDPRCS2Regs);
    1469        3490 :   emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false,
    1470             :               &isARMArea2Register, 0);
    1471        3490 :   emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false,
    1472             :               &isARMArea1Register, 0);
    1473             : 
    1474        3490 :   return true;
    1475             : }
    1476             : 
    1477             : // FIXME: Make generic?
    1478         589 : static unsigned GetFunctionSizeInBytes(const MachineFunction &MF,
    1479             :                                        const ARMBaseInstrInfo &TII) {
    1480             :   unsigned FnSize = 0;
    1481        1787 :   for (auto &MBB : MF) {
    1482        7759 :     for (auto &MI : MBB)
    1483        5363 :       FnSize += TII.getInstSizeInBytes(MI);
    1484             :   }
    1485         589 :   return FnSize;
    1486             : }
    1487             : 
    1488             : /// estimateRSStackSizeLimit - Look at each instruction that references stack
    1489             : /// frames and return the stack size limit beyond which some of these
    1490             : /// instructions will require a scratch register during their expansion later.
    1491             : // FIXME: Move to TII?
    1492       13666 : static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
    1493             :                                          const TargetFrameLowering *TFI) {
    1494       13666 :   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1495       13666 :   unsigned Limit = (1 << 12) - 1;
    1496       32769 :   for (auto &MBB : MF) {
    1497      153270 :     for (auto &MI : MBB) {
    1498      596878 :       for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    1499      991810 :         if (!MI.getOperand(i).isFI())
    1500             :           continue;
    1501             : 
    1502             :         // When using ADDri to get the address of a stack object, 255 is the
    1503             :         // largest offset guaranteed to fit in the immediate offset.
    1504       28140 :         if (MI.getOpcode() == ARM::ADDri) {
    1505        2486 :           Limit = std::min(Limit, (1U << 8) - 1);
    1506        1243 :           break;
    1507             :         }
    1508             : 
    1509             :         // Otherwise check the addressing mode.
    1510       12827 :         switch (MI.getDesc().TSFlags & ARMII::AddrModeMask) {
    1511          70 :         case ARMII::AddrMode3:
    1512             :         case ARMII::AddrModeT2_i8:
    1513         140 :           Limit = std::min(Limit, (1U << 8) - 1);
    1514          70 :           break;
    1515        1218 :         case ARMII::AddrMode5:
    1516             :         case ARMII::AddrModeT2_i8s4:
    1517        2436 :           Limit = std::min(Limit, ((1U << 8) - 1) * 4);
    1518        1218 :           break;
    1519        3065 :         case ARMII::AddrModeT2_i12:
    1520             :           // i12 supports only positive offset so these will be converted to
    1521             :           // i8 opcodes. See llvm::rewriteT2FrameIndex.
    1522        3065 :           if (TFI->hasFP(MF) && AFI->hasStackFrame())
    1523           0 :             Limit = std::min(Limit, (1U << 8) - 1);
    1524             :           break;
    1525          21 :         case ARMII::AddrMode4:
    1526             :         case ARMII::AddrMode6:
    1527             :           // Addressing modes 4 & 6 (load/store) instructions can't encode an
    1528             :           // immediate offset for stack references.
    1529          21 :           return 0;
    1530             :         default:
    1531             :           break;
    1532             :         }
    1533             :         break; // At most one FI per instruction
    1534             :       }
    1535             :     }
    1536             :   }
    1537             : 
    1538       13645 :   return Limit;
    1539             : }
    1540             : 
    1541             : // In functions that realign the stack, it can be an advantage to spill the
    1542             : // callee-saved vector registers after realigning the stack. The vst1 and vld1
    1543             : // instructions take alignment hints that can improve performance.
    1544             : static void
    1545       13666 : checkNumAlignedDPRCS2Regs(MachineFunction &MF, BitVector &SavedRegs) {
    1546       13666 :   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(0);
    1547       13666 :   if (!SpillAlignedNEONRegs)
    1548             :     return;
    1549             : 
    1550             :   // Naked functions don't spill callee-saved registers.
    1551       27326 :   if (MF.getFunction().hasFnAttribute(Attribute::Naked))
    1552             :     return;
    1553             : 
    1554             :   // We are planning to use NEON instructions vst1 / vld1.
    1555       13662 :   if (!static_cast<const ARMSubtarget &>(MF.getSubtarget()).hasNEON())
    1556             :     return;
    1557             : 
    1558             :   // Don't bother if the default stack alignment is sufficiently high.
    1559        7802 :   if (MF.getSubtarget().getFrameLowering()->getStackAlignment() >= 8)
    1560             :     return;
    1561             : 
    1562             :   // Aligned spills require stack realignment.
    1563        1775 :   if (!static_cast<const ARMBaseRegisterInfo *>(
    1564        1775 :            MF.getSubtarget().getRegisterInfo())->canRealignStack(MF))
    1565             :     return;
    1566             : 
    1567             :   // We always spill contiguous d-registers starting from d8. Count how many
    1568             :   // needs spilling.  The register allocator will almost always use the
    1569             :   // callee-saved registers in order, but it can happen that there are holes in
    1570             :   // the range.  Registers above the hole will be spilled to the standard DPRCS
    1571             :   // area.
    1572             :   unsigned NumSpills = 0;
    1573        1374 :   for (; NumSpills < 8; ++NumSpills)
    1574        2132 :     if (!SavedRegs.test(ARM::D8 + NumSpills))
    1575             :       break;
    1576             : 
    1577             :   // Don't do this for just one d-register. It's not worth it.
    1578         820 :   if (NumSpills < 2)
    1579             :     return;
    1580             : 
    1581             :   // Spill the first NumSpills D-registers after realigning the stack.
    1582          40 :   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(NumSpills);
    1583             : 
    1584             :   // A scratch register is required for the vst1 / vld1 instructions.
    1585             :   SavedRegs.set(ARM::R4);
    1586             : }
    1587             : 
    1588       13666 : void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
    1589             :                                             BitVector &SavedRegs,
    1590             :                                             RegScavenger *RS) const {
    1591       13666 :   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
    1592             :   // This tells PEI to spill the FP as if it is any other callee-save register
    1593             :   // to take advantage the eliminateFrameIndex machinery. This also ensures it
    1594             :   // is spilled in the order specified by getCalleeSavedRegs() to make it easier
    1595             :   // to combine multiple loads / stores.
    1596             :   bool CanEliminateFrame = true;
    1597             :   bool CS1Spilled = false;
    1598             :   bool LRSpilled = false;
    1599             :   unsigned NumGPRSpills = 0;
    1600             :   unsigned NumFPRSpills = 0;
    1601             :   SmallVector<unsigned, 4> UnspilledCS1GPRs;
    1602             :   SmallVector<unsigned, 4> UnspilledCS2GPRs;
    1603             :   const ARMBaseRegisterInfo *RegInfo = static_cast<const ARMBaseRegisterInfo *>(
    1604       13666 :       MF.getSubtarget().getRegisterInfo());
    1605             :   const ARMBaseInstrInfo &TII =
    1606       13666 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    1607       13666 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1608       13666 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1609       13666 :   MachineRegisterInfo &MRI = MF.getRegInfo();
    1610       13666 :   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1611             :   (void)TRI;  // Silence unused warning in non-assert builds.
    1612       13666 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
    1613             : 
    1614             :   // Spill R4 if Thumb2 function requires stack realignment - it will be used as
    1615             :   // scratch register. Also spill R4 if Thumb2 function has varsized objects,
    1616             :   // since it's not always possible to restore sp from fp in a single
    1617             :   // instruction.
    1618             :   // FIXME: It will be better just to find spare register here.
    1619       18171 :   if (AFI->isThumb2Function() &&
    1620        8993 :       (MFI.hasVarSizedObjects() || RegInfo->needsStackRealignment(MF)))
    1621             :     SavedRegs.set(ARM::R4);
    1622             : 
    1623             :   // If a stack probe will be emitted, spill R4 and LR, since they are
    1624             :   // clobbered by the stack probe call.
    1625             :   // This estimate should be a safe, conservative estimate. The actual
    1626             :   // stack probe is enabled based on the size of the local objects;
    1627             :   // this estimate also includes the varargs store size.
    1628       27494 :   if (STI.isTargetWindows() &&
    1629         162 :       WindowsRequiresStackProbe(MF, MFI.estimateStackSize(MF))) {
    1630             :     SavedRegs.set(ARM::R4);
    1631             :     SavedRegs.set(ARM::LR);
    1632             :   }
    1633             : 
    1634       13666 :   if (AFI->isThumb1OnlyFunction()) {
    1635             :     // Spill LR if Thumb1 function uses variable length argument lists.
    1636        1093 :     if (AFI->getArgRegsSaveSize() > 0)
    1637             :       SavedRegs.set(ARM::LR);
    1638             : 
    1639             :     // Spill R4 if Thumb1 epilogue has to restore SP from FP or the function
    1640             :     // requires stack alignment.  We don't know for sure what the stack size
    1641             :     // will be, but for this, an estimate is good enough. If there anything
    1642             :     // changes it, it'll be a spill, which implies we've used all the registers
    1643             :     // and so R4 is already used, so not marking it here will be OK.
    1644             :     // FIXME: It will be better just to find spare register here.
    1645        2133 :     if (MFI.hasVarSizedObjects() || RegInfo->needsStackRealignment(MF) ||
    1646        1040 :         MFI.estimateStackSize(MF) > 508)
    1647             :       SavedRegs.set(ARM::R4);
    1648             :   }
    1649             : 
    1650             :   // See if we can spill vector registers to aligned stack.
    1651       13666 :   checkNumAlignedDPRCS2Regs(MF, SavedRegs);
    1652             : 
    1653             :   // Spill the BasePtr if it's used.
    1654       13666 :   if (RegInfo->hasBasePointer(MF))
    1655          24 :     SavedRegs.set(RegInfo->getBaseRegister());
    1656             : 
    1657             :   // Don't spill FP if the frame can be eliminated. This is determined
    1658             :   // by scanning the callee-save registers to see if any is modified.
    1659       13666 :   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
    1660      474162 :   for (unsigned i = 0; CSRegs[i]; ++i) {
    1661      230248 :     unsigned Reg = CSRegs[i];
    1662             :     bool Spilled = false;
    1663      230248 :     if (SavedRegs.test(Reg)) {
    1664             :       Spilled = true;
    1665             :       CanEliminateFrame = false;
    1666             :     }
    1667             : 
    1668      570072 :     if (!ARM::GPRRegClass.contains(Reg)) {
    1669      109576 :       if (Spilled) {
    1670        2606 :         if (ARM::SPRRegClass.contains(Reg))
    1671           0 :           NumFPRSpills++;
    1672        2606 :         else if (ARM::DPRRegClass.contains(Reg))
    1673        1303 :           NumFPRSpills += 2;
    1674           0 :         else if (ARM::QPRRegClass.contains(Reg))
    1675           0 :           NumFPRSpills += 4;
    1676             :       }
    1677      321586 :       continue;
    1678             :     }
    1679             : 
    1680      120672 :     if (Spilled) {
    1681        8340 :       NumGPRSpills++;
    1682             : 
    1683       14382 :       if (!STI.splitFramePushPop(MF)) {
    1684        6042 :         if (Reg == ARM::LR)
    1685             :           LRSpilled = true;
    1686             :         CS1Spilled = true;
    1687        6042 :         continue;
    1688             :       }
    1689             : 
    1690             :       // Keep track if LR and any of R4, R5, R6, and R7 is spilled.
    1691        2298 :       switch (Reg) {
    1692         977 :       case ARM::LR:
    1693             :         LRSpilled = true;
    1694             :         LLVM_FALLTHROUGH;
    1695             :       case ARM::R0: case ARM::R1:
    1696             :       case ARM::R2: case ARM::R3:
    1697             :       case ARM::R4: case ARM::R5:
    1698             :       case ARM::R6: case ARM::R7:
    1699             :         CS1Spilled = true;
    1700             :         break;
    1701             :       default:
    1702             :         break;
    1703             :       }
    1704             :     } else {
    1705      208724 :       if (!STI.splitFramePushPop(MF)) {
    1706       96392 :         UnspilledCS1GPRs.push_back(Reg);
    1707       96392 :         continue;
    1708             :       }
    1709             : 
    1710       15940 :       switch (Reg) {
    1711        8693 :       case ARM::R0: case ARM::R1:
    1712             :       case ARM::R2: case ARM::R3:
    1713             :       case ARM::R4: case ARM::R5:
    1714             :       case ARM::R6: case ARM::R7:
    1715             :       case ARM::LR:
    1716        8693 :         UnspilledCS1GPRs.push_back(Reg);
    1717        8693 :         break;
    1718        7247 :       default:
    1719        7247 :         UnspilledCS2GPRs.push_back(Reg);
    1720        7247 :         break;
    1721             :       }
    1722             :     }
    1723             :   }
    1724             : 
    1725             :   bool ForceLRSpill = false;
    1726       13666 :   if (!LRSpilled && AFI->isThumb1OnlyFunction()) {
    1727         589 :     unsigned FnSize = GetFunctionSizeInBytes(MF, TII);
    1728             :     // Force LR to be spilled if the Thumb function size is > 2048. This enables
    1729             :     // use of BL to implement far jump. If it turns out that it's not needed
    1730             :     // then the branch fix up path will undo it.
    1731         589 :     if (FnSize >= (1 << 11)) {
    1732             :       CanEliminateFrame = false;
    1733             :       ForceLRSpill = true;
    1734             :     }
    1735             :   }
    1736             : 
    1737             :   // If any of the stack slot references may be out of range of an immediate
    1738             :   // offset, make sure a register (or a spill slot) is available for the
    1739             :   // register scavenger. Note that if we're indexing off the frame pointer, the
    1740             :   // effective stack size is 4 bytes larger since the FP points to the stack
    1741             :   // slot of the previous FP. Also, if we have variable sized objects in the
    1742             :   // function, stack slot references will often be negative, and some of
    1743             :   // our instructions are positive-offset only, so conservatively consider
    1744             :   // that case to want a spill slot (or register) as well. Similarly, if
    1745             :   // the function adjusts the stack pointer during execution and the
    1746             :   // adjustments aren't already part of our stack size estimate, our offset
    1747             :   // calculations may be off, so be conservative.
    1748             :   // FIXME: We could add logic to be more precise about negative offsets
    1749             :   //        and which instructions will need a scratch register for them. Is it
    1750             :   //        worth the effort and added fragility?
    1751             :   unsigned EstimatedStackSize =
    1752       13666 :       MFI.estimateStackSize(MF) + 4 * (NumGPRSpills + NumFPRSpills);
    1753             : 
    1754             :   // Determine biggest (positive) SP offset in MachineFrameInfo.
    1755       13666 :   int MaxFixedOffset = 0;
    1756       30462 :   for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
    1757        3130 :     int MaxObjectOffset = MFI.getObjectOffset(I) + MFI.getObjectSize(I);
    1758        1565 :     MaxFixedOffset = std::max(MaxFixedOffset, MaxObjectOffset);
    1759             :   }
    1760             : 
    1761       13666 :   bool HasFP = hasFP(MF);
    1762       13666 :   if (HasFP) {
    1763        1904 :     if (AFI->hasStackFrame())
    1764           3 :       EstimatedStackSize += 4;
    1765             :   } else {
    1766             :     // If FP is not used, SP will be used to access arguments, so count the
    1767             :     // size of arguments into the estimation.
    1768       11762 :     EstimatedStackSize += MaxFixedOffset;
    1769             :   }
    1770       13666 :   EstimatedStackSize += 16; // For possible paddings.
    1771             : 
    1772       13666 :   unsigned EstimatedRSStackSizeLimit = estimateRSStackSizeLimit(MF, this);
    1773             :   int MaxFPOffset = getMaxFPOffset(MF.getFunction(), *AFI);
    1774       13510 :   bool BigFrameOffsets = EstimatedStackSize >= EstimatedRSStackSizeLimit ||
    1775       26973 :     MFI.hasVarSizedObjects() ||
    1776       30620 :     (MFI.adjustsStack() && !canSimplifyCallFramePseudos(MF)) ||
    1777             :     // For large argument stacks fp relative addressed may overflow.
    1778        1797 :     (HasFP && (MaxFixedOffset - MaxFPOffset) >= (int)EstimatedRSStackSizeLimit);
    1779       13666 :   if (BigFrameOffsets ||
    1780       27332 :       !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF)) {
    1781             :     AFI->setHasStackFrame(true);
    1782             : 
    1783        4180 :     if (HasFP) {
    1784        1121 :       SavedRegs.set(FramePtr);
    1785             :       // If the frame pointer is required by the ABI, also spill LR so that we
    1786             :       // emit a complete frame record.
    1787        1121 :       if (MF.getTarget().Options.DisableFramePointerElim(MF) && !LRSpilled) {
    1788             :         SavedRegs.set(ARM::LR);
    1789             :         LRSpilled = true;
    1790         117 :         NumGPRSpills++;
    1791         234 :         auto LRPos = llvm::find(UnspilledCS1GPRs, ARM::LR);
    1792         117 :         if (LRPos != UnspilledCS1GPRs.end())
    1793         117 :           UnspilledCS1GPRs.erase(LRPos);
    1794             :       }
    1795             :       auto FPPos = llvm::find(UnspilledCS1GPRs, FramePtr);
    1796        1121 :       if (FPPos != UnspilledCS1GPRs.end())
    1797        1063 :         UnspilledCS1GPRs.erase(FPPos);
    1798        1121 :       NumGPRSpills++;
    1799        1121 :       if (FramePtr == ARM::R7)
    1800             :         CS1Spilled = true;
    1801             :     }
    1802             : 
    1803             :     // This is true when we inserted a spill for an unused register that can now
    1804             :     // be used for register scavenging.
    1805             :     bool ExtraCSSpill = false;
    1806             : 
    1807        4180 :     if (AFI->isThumb1OnlyFunction()) {
    1808             :       // For Thumb1-only targets, we need some low registers when we save and
    1809             :       // restore the high registers (which aren't allocatable, but could be
    1810             :       // used by inline assembly) because the push/pop instructions can not
    1811             :       // access high registers. If necessary, we might need to push more low
    1812             :       // registers to ensure that there is at least one free that can be used
    1813             :       // for the saving & restoring, and preferably we should ensure that as
    1814             :       // many as are needed are available so that fewer push/pop instructions
    1815             :       // are required.
    1816             : 
    1817             :       // Low registers which are not currently pushed, but could be (r4-r7).
    1818             :       SmallVector<unsigned, 4> AvailableRegs;
    1819             : 
    1820             :       // Unused argument registers (r0-r3) can be clobbered in the prologue for
    1821             :       // free.
    1822         619 :       int EntryRegDeficit = 0;
    1823        5571 :       for (unsigned Reg : {ARM::R0, ARM::R1, ARM::R2, ARM::R3}) {
    1824        2476 :         if (!MF.getRegInfo().isLiveIn(Reg)) {
    1825        1822 :           --EntryRegDeficit;
    1826             :           LLVM_DEBUG(dbgs()
    1827             :                      << printReg(Reg, TRI)
    1828             :                      << " is unused argument register, EntryRegDeficit = "
    1829             :                      << EntryRegDeficit << "\n");
    1830             :         }
    1831             :       }
    1832             : 
    1833             :       // Unused return registers can be clobbered in the epilogue for free.
    1834         619 :       int ExitRegDeficit = AFI->getReturnRegsCount() - 4;
    1835             :       LLVM_DEBUG(dbgs() << AFI->getReturnRegsCount()
    1836             :                         << " return regs used, ExitRegDeficit = "
    1837             :                         << ExitRegDeficit << "\n");
    1838             : 
    1839         619 :       int RegDeficit = std::max(EntryRegDeficit, ExitRegDeficit);
    1840             :       LLVM_DEBUG(dbgs() << "RegDeficit = " << RegDeficit << "\n");
    1841             : 
    1842             :       // r4-r6 can be used in the prologue if they are pushed by the first push
    1843             :       // instruction.
    1844        4333 :       for (unsigned Reg : {ARM::R4, ARM::R5, ARM::R6}) {
    1845        1857 :         if (SavedRegs.test(Reg)) {
    1846         601 :           --RegDeficit;
    1847             :           LLVM_DEBUG(dbgs() << printReg(Reg, TRI)
    1848             :                             << " is saved low register, RegDeficit = "
    1849             :                             << RegDeficit << "\n");
    1850             :         } else {
    1851        1256 :           AvailableRegs.push_back(Reg);
    1852             :           LLVM_DEBUG(
    1853             :               dbgs()
    1854             :               << printReg(Reg, TRI)
    1855             :               << " is non-saved low register, adding to AvailableRegs\n");
    1856             :         }
    1857             :       }
    1858             : 
    1859             :       // r7 can be used if it is not being used as the frame pointer.
    1860         619 :       if (!HasFP) {
    1861         462 :         if (SavedRegs.test(ARM::R7)) {
    1862          39 :           --RegDeficit;
    1863             :           LLVM_DEBUG(dbgs() << "%r7 is saved low register, RegDeficit = "
    1864             :                             << RegDeficit << "\n");
    1865             :         } else {
    1866         423 :           AvailableRegs.push_back(ARM::R7);
    1867             :           LLVM_DEBUG(
    1868             :               dbgs()
    1869             :               << "%r7 is non-saved low register, adding to AvailableRegs\n");
    1870             :         }
    1871             :       }
    1872             : 
    1873             :       // Each of r8-r11 needs to be copied to a low register, then pushed.
    1874        5571 :       for (unsigned Reg : {ARM::R8, ARM::R9, ARM::R10, ARM::R11}) {
    1875        2476 :         if (SavedRegs.test(Reg)) {
    1876          54 :           ++RegDeficit;
    1877             :           LLVM_DEBUG(dbgs() << printReg(Reg, TRI)
    1878             :                             << " is saved high register, RegDeficit = "
    1879             :                             << RegDeficit << "\n");
    1880             :         }
    1881             :       }
    1882             : 
    1883             :       // LR can only be used by PUSH, not POP, and can't be used at all if the
    1884             :       // llvm.returnaddress intrinsic is used. This is only worth doing if we
    1885             :       // are more limited at function entry than exit.
    1886         834 :       if ((EntryRegDeficit > ExitRegDeficit) &&
    1887         215 :           !(MF.getRegInfo().isLiveIn(ARM::LR) &&
    1888           0 :             MF.getFrameInfo().isReturnAddressTaken())) {
    1889         215 :         if (SavedRegs.test(ARM::LR)) {
    1890         170 :           --RegDeficit;
    1891             :           LLVM_DEBUG(dbgs() << "%lr is saved register, RegDeficit = "
    1892             :                             << RegDeficit << "\n");
    1893             :         } else {
    1894          45 :           AvailableRegs.push_back(ARM::LR);
    1895             :           LLVM_DEBUG(dbgs() << "%lr is not saved, adding to AvailableRegs\n");
    1896             :         }
    1897             :       }
    1898             : 
    1899             :       // If there are more high registers that need pushing than low registers
    1900             :       // available, push some more low registers so that we can use fewer push
    1901             :       // instructions. This might not reduce RegDeficit all the way to zero,
    1902             :       // because we can only guarantee that r4-r6 are available, but r8-r11 may
    1903             :       // need saving.
    1904             :       LLVM_DEBUG(dbgs() << "Final RegDeficit = " << RegDeficit << "\n");
    1905         635 :       for (; RegDeficit > 0 && !AvailableRegs.empty(); --RegDeficit) {
    1906           8 :         unsigned Reg = AvailableRegs.pop_back_val();
    1907             :         LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
    1908             :                           << " to make up reg deficit\n");
    1909             :         SavedRegs.set(Reg);
    1910           8 :         NumGPRSpills++;
    1911             :         CS1Spilled = true;
    1912             :         assert(!MRI.isReserved(Reg) && "Should not be reserved");
    1913           8 :         if (!MRI.isPhysRegUsed(Reg))
    1914             :           ExtraCSSpill = true;
    1915           8 :         UnspilledCS1GPRs.erase(llvm::find(UnspilledCS1GPRs, Reg));
    1916           8 :         if (Reg == ARM::LR)
    1917             :           LRSpilled = true;
    1918             :       }
    1919             :       LLVM_DEBUG(dbgs() << "After adding spills, RegDeficit = " << RegDeficit
    1920             :                         << "\n");
    1921             :     }
    1922             : 
    1923             :     // If LR is not spilled, but at least one of R4, R5, R6, and R7 is spilled.
    1924             :     // Spill LR as well so we can fold BX_RET to the registers restore (LDM).
    1925        4180 :     if (!LRSpilled && CS1Spilled) {
    1926             :       SavedRegs.set(ARM::LR);
    1927         189 :       NumGPRSpills++;
    1928             :       SmallVectorImpl<unsigned>::iterator LRPos;
    1929         378 :       LRPos = llvm::find(UnspilledCS1GPRs, (unsigned)ARM::LR);
    1930         189 :       if (LRPos != UnspilledCS1GPRs.end())
    1931         189 :         UnspilledCS1GPRs.erase(LRPos);
    1932             : 
    1933             :       ForceLRSpill = false;
    1934         189 :       if (!MRI.isReserved(ARM::LR) && !MRI.isPhysRegUsed(ARM::LR))
    1935             :         ExtraCSSpill = true;
    1936             :     }
    1937             : 
    1938             :     // If stack and double are 8-byte aligned and we are spilling an odd number
    1939             :     // of GPRs, spill one extra callee save GPR so we won't have to pad between
    1940             :     // the integer and double callee save areas.
    1941             :     LLVM_DEBUG(dbgs() << "NumGPRSpills = " << NumGPRSpills << "\n");
    1942        4180 :     unsigned TargetAlign = getStackAlignment();
    1943        4180 :     if (TargetAlign >= 8 && (NumGPRSpills & 1)) {
    1944        2038 :       if (CS1Spilled && !UnspilledCS1GPRs.empty()) {
    1945        3580 :         for (unsigned i = 0, e = UnspilledCS1GPRs.size(); i != e; ++i) {
    1946        7146 :           unsigned Reg = UnspilledCS1GPRs[i];
    1947             :           // Don't spill high register if the function is thumb.  In the case of
    1948             :           // Windows on ARM, accept R11 (frame pointer)
    1949        5972 :           if (!AFI->isThumbFunction() ||
    1950        2497 :               (STI.isTargetWindows() && Reg == ARM::R11) ||
    1951        5221 :               isARMLowRegister(Reg) || Reg == ARM::LR) {
    1952             :             SavedRegs.set(Reg);
    1953             :             LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
    1954             :                               << " to make up alignment\n");
    1955        1925 :             if (!MRI.isReserved(Reg) && !MRI.isPhysRegUsed(Reg))
    1956             :               ExtraCSSpill = true;
    1957             :             break;
    1958             :           }
    1959             :         }
    1960         153 :       } else if (!UnspilledCS2GPRs.empty() && !AFI->isThumb1OnlyFunction()) {
    1961           8 :         unsigned Reg = UnspilledCS2GPRs.front();
    1962             :         SavedRegs.set(Reg);
    1963             :         LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, TRI)
    1964             :                           << " to make up alignment\n");
    1965           8 :         if (!MRI.isReserved(Reg) && !MRI.isPhysRegUsed(Reg))
    1966             :           ExtraCSSpill = true;
    1967             :       }
    1968             :     }
    1969             : 
    1970             :     // Estimate if we might need to scavenge a register at some point in order
    1971             :     // to materialize a stack offset. If so, either spill one additional
    1972             :     // callee-saved register or reserve a special spill slot to facilitate
    1973             :     // register scavenging. Thumb1 needs a spill slot for stack pointer
    1974             :     // adjustments also, even when the frame itself is small.
    1975        4180 :     if (BigFrameOffsets && !ExtraCSSpill) {
    1976             :       // If any non-reserved CS register isn't spilled, just spill one or two
    1977             :       // extra. That should take care of it!
    1978         120 :       unsigned NumExtras = TargetAlign / 4;
    1979             :       SmallVector<unsigned, 2> Extras;
    1980         418 :       while (NumExtras && !UnspilledCS1GPRs.empty()) {
    1981         149 :         unsigned Reg = UnspilledCS1GPRs.back();
    1982             :         UnspilledCS1GPRs.pop_back();
    1983         149 :         if (!MRI.isReserved(Reg) &&
    1984         149 :             (!AFI->isThumb1OnlyFunction() || isARMLowRegister(Reg) ||
    1985             :              Reg == ARM::LR)) {
    1986         149 :           Extras.push_back(Reg);
    1987         149 :           NumExtras--;
    1988             :         }
    1989             :       }
    1990             :       // For non-Thumb1 functions, also check for hi-reg CS registers
    1991         120 :       if (!AFI->isThumb1OnlyFunction()) {
    1992          95 :         while (NumExtras && !UnspilledCS2GPRs.empty()) {
    1993           2 :           unsigned Reg = UnspilledCS2GPRs.back();
    1994             :           UnspilledCS2GPRs.pop_back();
    1995           2 :           if (!MRI.isReserved(Reg)) {
    1996           2 :             Extras.push_back(Reg);
    1997           2 :             NumExtras--;
    1998             :           }
    1999             :         }
    2000             :       }
    2001         120 :       if (NumExtras == 0) {
    2002         370 :         for (unsigned Reg : Extras) {
    2003             :           SavedRegs.set(Reg);
    2004         141 :           if (!MRI.isPhysRegUsed(Reg))
    2005             :             ExtraCSSpill = true;
    2006             :         }
    2007             :       }
    2008         120 :       if (!ExtraCSSpill && !AFI->isThumb1OnlyFunction()) {
    2009             :         // note: Thumb1 functions spill to R12, not the stack.  Reserve a slot
    2010             :         // closest to SP or frame pointer.
    2011             :         assert(RS && "Register scavenging not provided");
    2012             :         const TargetRegisterClass &RC = ARM::GPRRegClass;
    2013             :         unsigned Size = TRI->getSpillSize(RC);
    2014             :         unsigned Align = TRI->getSpillAlignment(RC);
    2015          17 :         RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
    2016             :       }
    2017             :     }
    2018             :   }
    2019             : 
    2020       13666 :   if (ForceLRSpill) {
    2021             :     SavedRegs.set(ARM::LR);
    2022             :     AFI->setLRIsSpilledForFarJump(true);
    2023             :   }
    2024       13666 : }
    2025             : 
    2026       13154 : MachineBasicBlock::iterator ARMFrameLowering::eliminateCallFramePseudoInstr(
    2027             :     MachineFunction &MF, MachineBasicBlock &MBB,
    2028             :     MachineBasicBlock::iterator I) const {
    2029             :   const ARMBaseInstrInfo &TII =
    2030       13154 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    2031       13154 :   if (!hasReservedCallFrame(MF)) {
    2032             :     // If we have alloca, convert as follows:
    2033             :     // ADJCALLSTACKDOWN -> sub, sp, sp, amount
    2034             :     // ADJCALLSTACKUP   -> add, sp, sp, amount
    2035             :     MachineInstr &Old = *I;
    2036             :     DebugLoc dl = Old.getDebugLoc();
    2037         164 :     unsigned Amount = TII.getFrameSize(Old);
    2038         164 :     if (Amount != 0) {
    2039             :       // We need to keep the stack aligned properly.  To do this, we round the
    2040             :       // amount of space needed for the outgoing arguments up to the next
    2041             :       // alignment boundary.
    2042          84 :       Amount = alignSPAdjust(Amount);
    2043             : 
    2044          42 :       ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    2045             :       assert(!AFI->isThumb1OnlyFunction() &&
    2046             :              "This eliminateCallFramePseudoInstr does not support Thumb1!");
    2047          42 :       bool isARM = !AFI->isThumbFunction();
    2048             : 
    2049             :       // Replace the pseudo instruction with a new instruction...
    2050          42 :       unsigned Opc = Old.getOpcode();
    2051          42 :       int PIdx = Old.findFirstPredOperandIdx();
    2052             :       ARMCC::CondCodes Pred =
    2053          84 :           (PIdx == -1) ? ARMCC::AL
    2054          84 :                        : (ARMCC::CondCodes)Old.getOperand(PIdx).getImm();
    2055             :       unsigned PredReg = TII.getFramePred(Old);
    2056          42 :       if (Opc == ARM::ADJCALLSTACKDOWN || Opc == ARM::tADJCALLSTACKDOWN) {
    2057          21 :         emitSPUpdate(isARM, MBB, I, dl, TII, -Amount, MachineInstr::NoFlags,
    2058             :                      Pred, PredReg);
    2059             :       } else {
    2060             :         assert(Opc == ARM::ADJCALLSTACKUP || Opc == ARM::tADJCALLSTACKUP);
    2061          21 :         emitSPUpdate(isARM, MBB, I, dl, TII, Amount, MachineInstr::NoFlags,
    2062             :                      Pred, PredReg);
    2063             :       }
    2064             :     }
    2065             :   }
    2066       13154 :   return MBB.erase(I);
    2067             : }
    2068             : 
    2069             : /// Get the minimum constant for ARM that is greater than or equal to the
    2070             : /// argument. In ARM, constants can have any value that can be produced by
    2071             : /// rotating an 8-bit value to the right by an even number of bits within a
    2072             : /// 32-bit word.
    2073         104 : static uint32_t alignToARMConstant(uint32_t Value) {
    2074             :   unsigned Shifted = 0;
    2075             : 
    2076         104 :   if (Value == 0)
    2077             :       return 0;
    2078             : 
    2079        1244 :   while (!(Value & 0xC0000000)) {
    2080         596 :       Value = Value << 2;
    2081         596 :       Shifted += 2;
    2082             :   }
    2083             : 
    2084          52 :   bool Carry = (Value & 0x00FFFFFF);
    2085          52 :   Value = ((Value & 0xFF000000) >> 24) + Carry;
    2086             : 
    2087          52 :   if (Value & 0x0000100)
    2088           0 :       Value = Value & 0x000001FC;
    2089             : 
    2090          52 :   if (Shifted > 24)
    2091          36 :       Value = Value >> (Shifted - 24);
    2092             :   else
    2093          16 :       Value = Value << (24 - Shifted);
    2094             : 
    2095             :   return Value;
    2096             : }
    2097             : 
    2098             : // The stack limit in the TCB is set to this many bytes above the actual
    2099             : // stack limit.
    2100             : static const uint64_t kSplitStackAvailable = 256;
    2101             : 
    2102             : // Adjust the function prologue to enable split stacks. This currently only
    2103             : // supports android and linux.
    2104             : //
    2105             : // The ABI of the segmented stack prologue is a little arbitrarily chosen, but
    2106             : // must be well defined in order to allow for consistent implementations of the
    2107             : // __morestack helper function. The ABI is also not a normal ABI in that it
    2108             : // doesn't follow the normal calling conventions because this allows the
    2109             : // prologue of each function to be optimized further.
    2110             : //
    2111             : // Currently, the ABI looks like (when calling __morestack)
    2112             : //
    2113             : //  * r4 holds the minimum stack size requested for this function call
    2114             : //  * r5 holds the stack size of the arguments to the function
    2115             : //  * the beginning of the function is 3 instructions after the call to
    2116             : //    __morestack
    2117             : //
    2118             : // Implementations of __morestack should use r4 to allocate a new stack, r5 to
    2119             : // place the arguments on to the new stack, and the 3-instruction knowledge to
    2120             : // jump directly to the body of the function when working on the new stack.
    2121             : //
    2122             : // An old (and possibly no longer compatible) implementation of __morestack for
    2123             : // ARM can be found at [1].
    2124             : //
    2125             : // [1] - https://github.com/mozilla/rust/blob/86efd9/src/rt/arch/arm/morestack.S
    2126          60 : void ARMFrameLowering::adjustForSegmentedStacks(
    2127             :     MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
    2128             :   unsigned Opcode;
    2129             :   unsigned CFIIndex;
    2130          60 :   const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
    2131          60 :   bool Thumb = ST->isThumb();
    2132             : 
    2133             :   // Sadly, this currently doesn't support varargs, platforms other than
    2134             :   // android/linux. Note that thumb1/thumb2 are support for android/linux.
    2135         120 :   if (MF.getFunction().isVarArg())
    2136           0 :     report_fatal_error("Segmented stacks do not support vararg functions.");
    2137          90 :   if (!ST->isTargetAndroid() && !ST->isTargetLinux())
    2138           0 :     report_fatal_error("Segmented stacks not supported on this platform.");
    2139             : 
    2140          60 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    2141          60 :   MachineModuleInfo &MMI = MF.getMMI();
    2142             :   MCContext &Context = MMI.getContext();
    2143          60 :   const MCRegisterInfo *MRI = Context.getRegisterInfo();
    2144             :   const ARMBaseInstrInfo &TII =
    2145          60 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    2146          60 :   ARMFunctionInfo *ARMFI = MF.getInfo<ARMFunctionInfo>();
    2147          60 :   DebugLoc DL;
    2148             : 
    2149          60 :   uint64_t StackSize = MFI.getStackSize();
    2150             : 
    2151             :   // Do not generate a prologue for functions with a stack of size zero
    2152          60 :   if (StackSize == 0)
    2153             :     return;
    2154             : 
    2155             :   // Use R4 and R5 as scratch registers.
    2156             :   // We save R4 and R5 before use and restore them before leaving the function.
    2157             :   unsigned ScratchReg0 = ARM::R4;
    2158             :   unsigned ScratchReg1 = ARM::R5;
    2159             :   uint64_t AlignedStackSize;
    2160             : 
    2161          52 :   MachineBasicBlock *PrevStackMBB = MF.CreateMachineBasicBlock();
    2162          52 :   MachineBasicBlock *PostStackMBB = MF.CreateMachineBasicBlock();
    2163          52 :   MachineBasicBlock *AllocMBB = MF.CreateMachineBasicBlock();
    2164          52 :   MachineBasicBlock *GetMBB = MF.CreateMachineBasicBlock();
    2165          52 :   MachineBasicBlock *McrMBB = MF.CreateMachineBasicBlock();
    2166             : 
    2167             :   // Grab everything that reaches PrologueMBB to update there liveness as well.
    2168             :   SmallPtrSet<MachineBasicBlock *, 8> BeforePrologueRegion;
    2169             :   SmallVector<MachineBasicBlock *, 2> WalkList;
    2170          52 :   WalkList.push_back(&PrologueMBB);
    2171             : 
    2172             :   do {
    2173             :     MachineBasicBlock *CurMBB = WalkList.pop_back_val();
    2174          52 :     for (MachineBasicBlock *PredBB : CurMBB->predecessors()) {
    2175           0 :       if (BeforePrologueRegion.insert(PredBB).second)
    2176           0 :         WalkList.push_back(PredBB);
    2177             :     }
    2178          52 :   } while (!WalkList.empty());
    2179             : 
    2180             :   // The order in that list is important.
    2181             :   // The blocks will all be inserted before PrologueMBB using that order.
    2182             :   // Therefore the block that should appear first in the CFG should appear
    2183             :   // first in the list.
    2184             :   MachineBasicBlock *AddedBlocks[] = {PrevStackMBB, McrMBB, GetMBB, AllocMBB,
    2185          52 :                                       PostStackMBB};
    2186             : 
    2187         572 :   for (MachineBasicBlock *B : AddedBlocks)
    2188         260 :     BeforePrologueRegion.insert(B);
    2189             : 
    2190         216 :   for (const auto &LI : PrologueMBB.liveins()) {
    2191         164 :     for (MachineBasicBlock *PredBB : BeforePrologueRegion)
    2192             :       PredBB->addLiveIn(LI);
    2193             :   }
    2194             : 
    2195             :   // Remove the newly added blocks from the list, since we know
    2196             :   // we do not have to do the following updates for them.
    2197         572 :   for (MachineBasicBlock *B : AddedBlocks) {
    2198             :     BeforePrologueRegion.erase(B);
    2199         260 :     MF.insert(PrologueMBB.getIterator(), B);
    2200             :   }
    2201             : 
    2202          52 :   for (MachineBasicBlock *MBB : BeforePrologueRegion) {
    2203             :     // Make sure the LiveIns are still sorted and unique.
    2204           0 :     MBB->sortUniqueLiveIns();
    2205             :     // Replace the edges to PrologueMBB by edges to the sequences
    2206             :     // we are about to add.
    2207           0 :     MBB->ReplaceUsesOfBlockWith(&PrologueMBB, AddedBlocks[0]);
    2208             :   }
    2209             : 
    2210             :   // The required stack size that is aligned to ARM constant criterion.
    2211          52 :   AlignedStackSize = alignToARMConstant(StackSize);
    2212             : 
    2213             :   // When the frame size is less than 256 we just compare the stack
    2214             :   // boundary directly to the value of the stack pointer, per gcc.
    2215             :   bool CompareStackPointer = AlignedStackSize < kSplitStackAvailable;
    2216             : 
    2217             :   // We will use two of the callee save registers as scratch registers so we
    2218             :   // need to save those registers onto the stack.
    2219             :   // We will use SR0 to hold stack limit and SR1 to hold the stack size
    2220             :   // requested and arguments for __morestack().
    2221             :   // SR0: Scratch Register #0
    2222             :   // SR1: Scratch Register #1
    2223             :   // push {SR0, SR1}
    2224          52 :   if (Thumb) {
    2225          78 :     BuildMI(PrevStackMBB, DL, TII.get(ARM::tPUSH))
    2226          26 :         .add(predOps(ARMCC::AL))
    2227          26 :         .addReg(ScratchReg0)
    2228          26 :         .addReg(ScratchReg1);
    2229             :   } else {
    2230          78 :     BuildMI(PrevStackMBB, DL, TII.get(ARM::STMDB_UPD))
    2231          26 :         .addReg(ARM::SP, RegState::Define)
    2232          26 :         .addReg(ARM::SP)
    2233          26 :         .add(predOps(ARMCC::AL))
    2234          26 :         .addReg(ScratchReg0)
    2235          26 :         .addReg(ScratchReg1);
    2236             :   }
    2237             : 
    2238             :   // Emit the relevant DWARF information about the change in stack pointer as
    2239             :   // well as where to find both r4 and r5 (the callee-save registers)
    2240             :   CFIIndex =
    2241          52 :       MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, -8));
    2242          52 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2243             :       .addCFIIndex(CFIIndex);
    2244         104 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2245          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg1, true), -4));
    2246          52 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2247             :       .addCFIIndex(CFIIndex);
    2248         104 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2249          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg0, true), -8));
    2250          52 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2251             :       .addCFIIndex(CFIIndex);
    2252             : 
    2253             :   // mov SR1, sp
    2254          52 :   if (Thumb) {
    2255          78 :     BuildMI(McrMBB, DL, TII.get(ARM::tMOVr), ScratchReg1)
    2256          26 :         .addReg(ARM::SP)
    2257          26 :         .add(predOps(ARMCC::AL));
    2258          26 :   } else if (CompareStackPointer) {
    2259          54 :     BuildMI(McrMBB, DL, TII.get(ARM::MOVr), ScratchReg1)
    2260          18 :         .addReg(ARM::SP)
    2261          18 :         .add(predOps(ARMCC::AL))
    2262          18 :         .add(condCodeOp());
    2263             :   }
    2264             : 
    2265             :   // sub SR1, sp, #StackSize
    2266          52 :   if (!CompareStackPointer && Thumb) {
    2267          24 :     BuildMI(McrMBB, DL, TII.get(ARM::tSUBi8), ScratchReg1)
    2268           8 :         .add(condCodeOp())
    2269           8 :         .addReg(ScratchReg1)
    2270           8 :         .addImm(AlignedStackSize)
    2271           8 :         .add(predOps(ARMCC::AL));
    2272          44 :   } else if (!CompareStackPointer) {
    2273          24 :     BuildMI(McrMBB, DL, TII.get(ARM::SUBri), ScratchReg1)
    2274           8 :         .addReg(ARM::SP)
    2275           8 :         .addImm(AlignedStackSize)
    2276           8 :         .add(predOps(ARMCC::AL))
    2277           8 :         .add(condCodeOp());
    2278             :   }
    2279             : 
    2280          52 :   if (Thumb && ST->isThumb1Only()) {
    2281             :     unsigned PCLabelId = ARMFI->createPICLabelUId();
    2282          48 :     ARMConstantPoolValue *NewCPV = ARMConstantPoolSymbol::Create(
    2283          48 :         MF.getFunction().getContext(), "__STACK_LIMIT", PCLabelId, 0);
    2284          24 :     MachineConstantPool *MCP = MF.getConstantPool();
    2285          24 :     unsigned CPI = MCP->getConstantPoolIndex(NewCPV, 4);
    2286             : 
    2287             :     // ldr SR0, [pc, offset(STACK_LIMIT)]
    2288          24 :     BuildMI(GetMBB, DL, TII.get(ARM::tLDRpci), ScratchReg0)
    2289             :         .addConstantPoolIndex(CPI)
    2290          24 :         .add(predOps(ARMCC::AL));
    2291             : 
    2292             :     // ldr SR0, [SR0]
    2293          72 :     BuildMI(GetMBB, DL, TII.get(ARM::tLDRi), ScratchReg0)
    2294          24 :         .addReg(ScratchReg0)
    2295             :         .addImm(0)
    2296          24 :         .add(predOps(ARMCC::AL));
    2297             :   } else {
    2298             :     // Get TLS base address from the coprocessor
    2299             :     // mrc p15, #0, SR0, c13, c0, #3
    2300          28 :     BuildMI(McrMBB, DL, TII.get(ARM::MRC), ScratchReg0)
    2301             :         .addImm(15)
    2302             :         .addImm(0)
    2303             :         .addImm(13)
    2304             :         .addImm(0)
    2305             :         .addImm(3)
    2306          28 :         .add(predOps(ARMCC::AL));
    2307             : 
    2308             :     // Use the last tls slot on android and a private field of the TCP on linux.
    2309             :     assert(ST->isTargetAndroid() || ST->isTargetLinux());
    2310          28 :     unsigned TlsOffset = ST->isTargetAndroid() ? 63 : 1;
    2311             : 
    2312             :     // Get the stack limit from the right offset
    2313             :     // ldr SR0, [sr0, #4 * TlsOffset]
    2314          84 :     BuildMI(GetMBB, DL, TII.get(ARM::LDRi12), ScratchReg0)
    2315          28 :         .addReg(ScratchReg0)
    2316          28 :         .addImm(4 * TlsOffset)
    2317          28 :         .add(predOps(ARMCC::AL));
    2318             :   }
    2319             : 
    2320             :   // Compare stack limit with stack size requested.
    2321             :   // cmp SR0, SR1
    2322          52 :   Opcode = Thumb ? ARM::tCMPr : ARM::CMPrr;
    2323         156 :   BuildMI(GetMBB, DL, TII.get(Opcode))
    2324          52 :       .addReg(ScratchReg0)
    2325          52 :       .addReg(ScratchReg1)
    2326          52 :       .add(predOps(ARMCC::AL));
    2327             : 
    2328             :   // This jump is taken if StackLimit < SP - stack required.
    2329          52 :   Opcode = Thumb ? ARM::tBcc : ARM::Bcc;
    2330         156 :   BuildMI(GetMBB, DL, TII.get(Opcode)).addMBB(PostStackMBB)
    2331             :        .addImm(ARMCC::LO)
    2332          52 :        .addReg(ARM::CPSR);
    2333             : 
    2334             : 
    2335             :   // Calling __morestack(StackSize, Size of stack arguments).
    2336             :   // __morestack knows that the stack size requested is in SR0(r4)
    2337             :   // and amount size of stack arguments is in SR1(r5).
    2338             : 
    2339             :   // Pass first argument for the __morestack by Scratch Register #0.
    2340             :   //   The amount size of stack required
    2341          52 :   if (Thumb) {
    2342          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tMOVi8), ScratchReg0)
    2343          26 :         .add(condCodeOp())
    2344          26 :         .addImm(AlignedStackSize)
    2345          26 :         .add(predOps(ARMCC::AL));
    2346             :   } else {
    2347          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::MOVi), ScratchReg0)
    2348          26 :         .addImm(AlignedStackSize)
    2349          26 :         .add(predOps(ARMCC::AL))
    2350          26 :         .add(condCodeOp());
    2351             :   }
    2352             :   // Pass second argument for the __morestack by Scratch Register #1.
    2353             :   //   The amount size of stack consumed to save function arguments.
    2354          52 :   if (Thumb) {
    2355          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tMOVi8), ScratchReg1)
    2356          26 :         .add(condCodeOp())
    2357          26 :         .addImm(alignToARMConstant(ARMFI->getArgumentStackSize()))
    2358          26 :         .add(predOps(ARMCC::AL));
    2359             :   } else {
    2360          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::MOVi), ScratchReg1)
    2361          26 :         .addImm(alignToARMConstant(ARMFI->getArgumentStackSize()))
    2362          26 :         .add(predOps(ARMCC::AL))
    2363          26 :         .add(condCodeOp());
    2364             :   }
    2365             : 
    2366             :   // push {lr} - Save return address of this function.
    2367          52 :   if (Thumb) {
    2368          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::tPUSH))
    2369          26 :         .add(predOps(ARMCC::AL))
    2370          26 :         .addReg(ARM::LR);
    2371             :   } else {
    2372          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::STMDB_UPD))
    2373          26 :         .addReg(ARM::SP, RegState::Define)
    2374          26 :         .addReg(ARM::SP)
    2375          26 :         .add(predOps(ARMCC::AL))
    2376          26 :         .addReg(ARM::LR);
    2377             :   }
    2378             : 
    2379             :   // Emit the DWARF info about the change in stack as well as where to find the
    2380             :   // previous link register
    2381             :   CFIIndex =
    2382          52 :       MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, -12));
    2383          52 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2384             :       .addCFIIndex(CFIIndex);
    2385         104 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2386          52 :         nullptr, MRI->getDwarfRegNum(ARM::LR, true), -12));
    2387          52 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2388             :       .addCFIIndex(CFIIndex);
    2389             : 
    2390             :   // Call __morestack().
    2391          52 :   if (Thumb) {
    2392          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::tBL))
    2393          26 :         .add(predOps(ARMCC::AL))
    2394             :         .addExternalSymbol("__morestack");
    2395             :   } else {
    2396          26 :     BuildMI(AllocMBB, DL, TII.get(ARM::BL))
    2397             :         .addExternalSymbol("__morestack");
    2398             :   }
    2399             : 
    2400             :   // pop {lr} - Restore return address of this original function.
    2401          52 :   if (Thumb) {
    2402          26 :     if (ST->isThumb1Only()) {
    2403          72 :       BuildMI(AllocMBB, DL, TII.get(ARM::tPOP))
    2404          24 :           .add(predOps(ARMCC::AL))
    2405          24 :           .addReg(ScratchReg0);
    2406          72 :       BuildMI(AllocMBB, DL, TII.get(ARM::tMOVr), ARM::LR)
    2407          24 :           .addReg(ScratchReg0)
    2408          24 :           .add(predOps(ARMCC::AL));
    2409             :     } else {
    2410           6 :       BuildMI(AllocMBB, DL, TII.get(ARM::t2LDR_POST))
    2411           2 :           .addReg(ARM::LR, RegState::Define)
    2412           2 :           .addReg(ARM::SP, RegState::Define)
    2413           2 :           .addReg(ARM::SP)
    2414             :           .addImm(4)
    2415           2 :           .add(predOps(ARMCC::AL));
    2416             :     }
    2417             :   } else {
    2418          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::LDMIA_UPD))
    2419          26 :         .addReg(ARM::SP, RegState::Define)
    2420          26 :         .addReg(ARM::SP)
    2421          26 :         .add(predOps(ARMCC::AL))
    2422          26 :         .addReg(ARM::LR);
    2423             :   }
    2424             : 
    2425             :   // Restore SR0 and SR1 in case of __morestack() was called.
    2426             :   // __morestack() will skip PostStackMBB block so we need to restore
    2427             :   // scratch registers from here.
    2428             :   // pop {SR0, SR1}
    2429          52 :   if (Thumb) {
    2430          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::tPOP))
    2431          26 :         .add(predOps(ARMCC::AL))
    2432          26 :         .addReg(ScratchReg0)
    2433          26 :         .addReg(ScratchReg1);
    2434             :   } else {
    2435          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::LDMIA_UPD))
    2436          26 :         .addReg(ARM::SP, RegState::Define)
    2437          26 :         .addReg(ARM::SP)
    2438          26 :         .add(predOps(ARMCC::AL))
    2439          26 :         .addReg(ScratchReg0)
    2440          26 :         .addReg(ScratchReg1);
    2441             :   }
    2442             : 
    2443             :   // Update the CFA offset now that we've popped
    2444          52 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, 0));
    2445          52 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2446             :       .addCFIIndex(CFIIndex);
    2447             : 
    2448             :   // Return from this function.
    2449         104 :   BuildMI(AllocMBB, DL, TII.get(ST->getReturnOpcode())).add(predOps(ARMCC::AL));
    2450             : 
    2451             :   // Restore SR0 and SR1 in case of __morestack() was not called.
    2452             :   // pop {SR0, SR1}
    2453          52 :   if (Thumb) {
    2454          78 :     BuildMI(PostStackMBB, DL, TII.get(ARM::tPOP))
    2455          26 :         .add(predOps(ARMCC::AL))
    2456          26 :         .addReg(ScratchReg0)
    2457          26 :         .addReg(ScratchReg1);
    2458             :   } else {
    2459          78 :     BuildMI(PostStackMBB, DL, TII.get(ARM::LDMIA_UPD))
    2460          26 :         .addReg(ARM::SP, RegState::Define)
    2461          26 :         .addReg(ARM::SP)
    2462          26 :         .add(predOps(ARMCC::AL))
    2463          26 :         .addReg(ScratchReg0)
    2464          26 :         .addReg(ScratchReg1);
    2465             :   }
    2466             : 
    2467             :   // Update the CFA offset now that we've popped
    2468          52 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, 0));
    2469          52 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2470             :       .addCFIIndex(CFIIndex);
    2471             : 
    2472             :   // Tell debuggers that r4 and r5 are now the same as they were in the
    2473             :   // previous function, that they're the "Same Value".
    2474         104 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
    2475          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg0, true)));
    2476          52 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2477             :       .addCFIIndex(CFIIndex);
    2478         104 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
    2479          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg1, true)));
    2480          52 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2481             :       .addCFIIndex(CFIIndex);
    2482             : 
    2483             :   // Organizing MBB lists
    2484          52 :   PostStackMBB->addSuccessor(&PrologueMBB);
    2485             : 
    2486          52 :   AllocMBB->addSuccessor(PostStackMBB);
    2487             : 
    2488          52 :   GetMBB->addSuccessor(PostStackMBB);
    2489          52 :   GetMBB->addSuccessor(AllocMBB);
    2490             : 
    2491          52 :   McrMBB->addSuccessor(GetMBB);
    2492             : 
    2493          52 :   PrevStackMBB->addSuccessor(McrMBB);
    2494             : 
    2495             : #ifdef EXPENSIVE_CHECKS
    2496             :   MF.verify();
    2497             : #endif
    2498      303507 : }

Generated by: LCOV version 1.13