LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMFrameLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1137 1161 97.9 %
Date: 2017-09-14 15:23:50 Functions: 31 31 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- ARMFrameLowering.cpp - ARM Frame Information ----------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the ARM implementation of TargetFrameLowering class.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "ARMFrameLowering.h"
      15             : #include "ARMBaseInstrInfo.h"
      16             : #include "ARMBaseRegisterInfo.h"
      17             : #include "ARMConstantPoolValue.h"
      18             : #include "ARMMachineFunctionInfo.h"
      19             : #include "ARMSubtarget.h"
      20             : #include "MCTargetDesc/ARMAddressingModes.h"
      21             : #include "MCTargetDesc/ARMBaseInfo.h"
      22             : #include "llvm/ADT/BitVector.h"
      23             : #include "llvm/ADT/STLExtras.h"
      24             : #include "llvm/ADT/SmallPtrSet.h"
      25             : #include "llvm/ADT/SmallVector.h"
      26             : #include "llvm/CodeGen/MachineBasicBlock.h"
      27             : #include "llvm/CodeGen/MachineConstantPool.h"
      28             : #include "llvm/CodeGen/MachineFrameInfo.h"
      29             : #include "llvm/CodeGen/MachineFunction.h"
      30             : #include "llvm/CodeGen/MachineInstr.h"
      31             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      32             : #include "llvm/CodeGen/MachineModuleInfo.h"
      33             : #include "llvm/CodeGen/MachineOperand.h"
      34             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      35             : #include "llvm/CodeGen/RegisterScavenging.h"
      36             : #include "llvm/IR/Attributes.h"
      37             : #include "llvm/IR/CallingConv.h"
      38             : #include "llvm/IR/DebugLoc.h"
      39             : #include "llvm/IR/Function.h"
      40             : #include "llvm/MC/MCContext.h"
      41             : #include "llvm/MC/MCDwarf.h"
      42             : #include "llvm/MC/MCRegisterInfo.h"
      43             : #include "llvm/Support/CodeGen.h"
      44             : #include "llvm/Support/CommandLine.h"
      45             : #include "llvm/Support/Compiler.h"
      46             : #include "llvm/Support/Debug.h"
      47             : #include "llvm/Support/ErrorHandling.h"
      48             : #include "llvm/Support/MathExtras.h"
      49             : #include "llvm/Support/raw_ostream.h"
      50             : #include "llvm/Target/TargetInstrInfo.h"
      51             : #include "llvm/Target/TargetMachine.h"
      52             : #include "llvm/Target/TargetOptions.h"
      53             : #include "llvm/Target/TargetRegisterInfo.h"
      54             : #include "llvm/Target/TargetSubtargetInfo.h"
      55             : #include <algorithm>
      56             : #include <cassert>
      57             : #include <cstddef>
      58             : #include <cstdint>
      59             : #include <iterator>
      60             : #include <utility>
      61             : #include <vector>
      62             : 
      63             : #define DEBUG_TYPE "arm-frame-lowering"
      64             : 
      65             : using namespace llvm;
      66             : 
      67             : static cl::opt<bool>
      68      289224 : SpillAlignedNEONRegs("align-neon-spills", cl::Hidden, cl::init(true),
      69      289224 :                      cl::desc("Align ARM NEON spills in prolog and epilog"));
      70             : 
      71             : static MachineBasicBlock::iterator
      72             : skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
      73             :                         unsigned NumAlignedDPRCS2Regs);
      74             : 
      75        4487 : ARMFrameLowering::ARMFrameLowering(const ARMSubtarget &sti)
      76             :     : TargetFrameLowering(StackGrowsDown, sti.getStackAlignment(), 0, 4),
      77        8974 :       STI(sti) {}
      78             : 
      79      301909 : bool ARMFrameLowering::noFramePointerElim(const MachineFunction &MF) const {
      80             :   // iOS always has a FP for backtracking, force other targets to keep their FP
      81             :   // when doing FastISel. The emitted code is currently superior, and in cases
      82             :   // like test-suite's lencod FastISel isn't quite correct when FP is eliminated.
      83      570678 :   return TargetFrameLowering::noFramePointerElim(MF) ||
      84      570678 :          MF.getSubtarget<ARMSubtarget>().useFastISel();
      85             : }
      86             : 
      87             : /// hasFP - Return true if the specified function should have a dedicated frame
      88             : /// pointer register.  This is true if the function has variable sized allocas
      89             : /// or if frame pointer elimination is disabled.
      90      123088 : bool ARMFrameLowering::hasFP(const MachineFunction &MF) const {
      91      123088 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
      92      123088 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
      93             : 
      94             :   // ABI-required frame pointer.
      95      123088 :   if (MF.getTarget().Options.DisableFramePointerElim(MF))
      96             :     return true;
      97             : 
      98             :   // Frame pointer required for use within this function.
      99      200115 :   return (RegInfo->needsStackRealignment(MF) ||
     100      199521 :           MFI.hasVarSizedObjects() ||
     101             :           MFI.isFrameAddressTaken());
     102             : }
     103             : 
     104             : /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
     105             : /// not required, we reserve argument space for call sites in the function
     106             : /// immediately on entry to the current function.  This eliminates the need for
     107             : /// add/sub sp brackets around call sites.  Returns true if the call frame is
     108             : /// included as part of the stack frame.
     109       57690 : bool ARMFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
     110       57690 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     111       57689 :   unsigned CFSize = MFI.getMaxCallFrameSize();
     112             :   // It's not always a good idea to include the call frame as part of the
     113             :   // stack frame. ARM (especially Thumb) has small immediate offset to
     114             :   // address the stack frame. So a large call frame can cause poor codegen
     115             :   // and may even makes it impossible to scavenge a register.
     116       57689 :   if (CFSize >= ((1 << 12) - 1) / 2)  // Half of imm12
     117             :     return false;
     118             : 
     119       56471 :   return !MFI.hasVarSizedObjects();
     120             : }
     121             : 
     122             : /// canSimplifyCallFramePseudos - If there is a reserved call frame, the
     123             : /// call frame pseudos can be simplified.  Unlike most targets, having a FP
     124             : /// is not sufficient here since we still may reference some objects via SP
     125             : /// even when FP is available in Thumb2 mode.
     126             : bool
     127       16542 : ARMFrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
     128       16542 :   return hasReservedCallFrame(MF) || MF.getFrameInfo().hasVarSizedObjects();
     129             : }
     130             : 
     131        7022 : static bool isCSRestore(MachineInstr &MI, const ARMBaseInstrInfo &TII,
     132             :                         const MCPhysReg *CSRegs) {
     133             :   // Integer spill area is handled with "pop".
     134       20040 :   if (isPopOpcode(MI.getOpcode())) {
     135             :     // The first two operands are predicates. The last two are
     136             :     // imp-def and imp-use of SP. Check everything in between.
     137        3312 :     for (int i = 5, e = MI.getNumOperands(); i != e; ++i)
     138        6858 :       if (!isCalleeSavedRegister(MI.getOperand(i).getReg(), CSRegs))
     139             :         return false;
     140             :     return true;
     141             :   }
     142       11803 :   if ((MI.getOpcode() == ARM::LDR_POST_IMM ||
     143       11614 :        MI.getOpcode() == ARM::LDR_POST_REG ||
     144        6067 :        MI.getOpcode() == ARM::t2LDR_POST) &&
     145        6776 :       isCalleeSavedRegister(MI.getOperand(0).getReg(), CSRegs) &&
     146         260 :       MI.getOperand(1).getReg() == ARM::SP)
     147             :     return true;
     148             : 
     149             :   return false;
     150             : }
     151             : 
     152        3429 : static void emitRegPlusImmediate(
     153             :     bool isARM, MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
     154             :     const DebugLoc &dl, const ARMBaseInstrInfo &TII, unsigned DestReg,
     155             :     unsigned SrcReg, int NumBytes, unsigned MIFlags = MachineInstr::NoFlags,
     156             :     ARMCC::CondCodes Pred = ARMCC::AL, unsigned PredReg = 0) {
     157        3429 :   if (isARM)
     158        2253 :     emitARMRegPlusImmediate(MBB, MBBI, dl, DestReg, SrcReg, NumBytes,
     159             :                             Pred, PredReg, TII, MIFlags);
     160             :   else
     161        1176 :     emitT2RegPlusImmediate(MBB, MBBI, dl, DestReg, SrcReg, NumBytes,
     162             :                            Pred, PredReg, TII, MIFlags);
     163        3429 : }
     164             : 
     165             : static void emitSPUpdate(bool isARM, MachineBasicBlock &MBB,
     166             :                          MachineBasicBlock::iterator &MBBI, const DebugLoc &dl,
     167             :                          const ARMBaseInstrInfo &TII, int NumBytes,
     168             :                          unsigned MIFlags = MachineInstr::NoFlags,
     169             :                          ARMCC::CondCodes Pred = ARMCC::AL,
     170             :                          unsigned PredReg = 0) {
     171        2477 :   emitRegPlusImmediate(isARM, MBB, MBBI, dl, TII, ARM::SP, ARM::SP, NumBytes,
     172             :                        MIFlags, Pred, PredReg);
     173             : }
     174             : 
     175        1156 : static int sizeOfSPAdjustment(const MachineInstr &MI) {
     176             :   int RegSize;
     177        2312 :   switch (MI.getOpcode()) {
     178             :   case ARM::VSTMDDB_UPD:
     179             :     RegSize = 8;
     180             :     break;
     181         944 :   case ARM::STMDB_UPD:
     182             :   case ARM::t2STMDB_UPD:
     183         944 :     RegSize = 4;
     184         944 :     break;
     185             :   case ARM::t2STR_PRE:
     186             :   case ARM::STR_PRE_IMM:
     187             :     return 4;
     188           0 :   default:
     189           0 :     llvm_unreachable("Unknown push or pop like instruction");
     190             :   }
     191             : 
     192        1148 :   int count = 0;
     193             :   // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
     194             :   // pred) so the list starts at 4.
     195        5176 :   for (int i = MI.getNumOperands() - 1; i >= 4; --i)
     196        4028 :     count += RegSize;
     197             :   return count;
     198             : }
     199             : 
     200         146 : static bool WindowsRequiresStackProbe(const MachineFunction &MF,
     201             :                                       size_t StackSizeInBytes) {
     202         146 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     203         146 :   const Function *F = MF.getFunction();
     204         146 :   unsigned StackProbeSize = (MFI.getStackProtectorIndex() > 0) ? 4080 : 4096;
     205         292 :   if (F->hasFnAttribute("stack-probe-size"))
     206           6 :     F->getFnAttribute("stack-probe-size")
     207           4 :         .getValueAsString()
     208           2 :         .getAsInteger(0, StackProbeSize);
     209         146 :   return StackSizeInBytes >= StackProbeSize;
     210             : }
     211             : 
     212             : namespace {
     213             : 
     214       44248 : struct StackAdjustingInsts {
     215             :   struct InstInfo {
     216             :     MachineBasicBlock::iterator I;
     217             :     unsigned SPAdjust;
     218             :     bool BeforeFPSet;
     219             :   };
     220             : 
     221             :   SmallVector<InstInfo, 4> Insts;
     222             : 
     223             :   void addInst(MachineBasicBlock::iterator I, unsigned SPAdjust,
     224             :                bool BeforeFPSet = false) {
     225        5008 :     InstInfo Info = {I, SPAdjust, BeforeFPSet};
     226        5008 :     Insts.push_back(Info);
     227             :   }
     228             : 
     229             :   void addExtraBytes(const MachineBasicBlock::iterator I, unsigned ExtraBytes) {
     230             :     auto Info =
     231         114 :         llvm::find_if(Insts, [&](InstInfo &Info) { return Info.I == I; });
     232             :     assert(Info != Insts.end() && "invalid sp adjusting instruction");
     233          22 :     Info->SPAdjust += ExtraBytes;
     234             :   }
     235             : 
     236       11062 :   void emitDefCFAOffsets(MachineBasicBlock &MBB, const DebugLoc &dl,
     237             :                          const ARMBaseInstrInfo &TII, bool HasFP) {
     238       11062 :     MachineFunction &MF = *MBB.getParent();
     239       11062 :     unsigned CFAOffset = 0;
     240       37529 :     for (auto &Info : Insts) {
     241        4882 :       if (HasFP && !Info.BeforeFPSet)
     242             :         return;
     243             : 
     244        4343 :       CFAOffset -= Info.SPAdjust;
     245             :       unsigned CFIIndex = MF.addFrameInst(
     246       17372 :           MCCFIInstruction::createDefCfaOffset(nullptr, CFAOffset));
     247        8686 :       BuildMI(MBB, std::next(Info.I), dl,
     248       13029 :               TII.get(TargetOpcode::CFI_INSTRUCTION))
     249        4343 :               .addCFIIndex(CFIIndex)
     250        4343 :               .setMIFlags(MachineInstr::FrameSetup);
     251             :     }
     252             :   }
     253             : };
     254             : 
     255             : } // end anonymous namespace
     256             : 
     257             : /// Emit an instruction sequence that will align the address in
     258             : /// register Reg by zero-ing out the lower bits.  For versions of the
     259             : /// architecture that support Neon, this must be done in a single
     260             : /// instruction, since skipAlignedDPRCS2Spills assumes it is done in a
     261             : /// single instruction. That function only gets called when optimizing
     262             : /// spilling of D registers on a core with the Neon instruction set
     263             : /// present.
     264         204 : static void emitAligningInstructions(MachineFunction &MF, ARMFunctionInfo *AFI,
     265             :                                      const TargetInstrInfo &TII,
     266             :                                      MachineBasicBlock &MBB,
     267             :                                      MachineBasicBlock::iterator MBBI,
     268             :                                      const DebugLoc &DL, const unsigned Reg,
     269             :                                      const unsigned Alignment,
     270             :                                      const bool MustBeSingleInstruction) {
     271             :   const ARMSubtarget &AST =
     272         204 :       static_cast<const ARMSubtarget &>(MF.getSubtarget());
     273         204 :   const bool CanUseBFC = AST.hasV6T2Ops() || AST.hasV7Ops();
     274         204 :   const unsigned AlignMask = Alignment - 1;
     275         204 :   const unsigned NrBitsToZero = countTrailingZeros(Alignment);
     276             :   assert(!AFI->isThumb1OnlyFunction() && "Thumb1 not supported");
     277         204 :   if (!AFI->isThumbFunction()) {
     278             :     // if the BFC instruction is available, use that to zero the lower
     279             :     // bits:
     280             :     //   bfc Reg, #0, log2(Alignment)
     281             :     // otherwise use BIC, if the mask to zero the required number of bits
     282             :     // can be encoded in the bic immediate field
     283             :     //   bic Reg, Reg, Alignment-1
     284             :     // otherwise, emit
     285             :     //   lsr Reg, Reg, log2(Alignment)
     286             :     //   lsl Reg, Reg, log2(Alignment)
     287         115 :     if (CanUseBFC) {
     288         297 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::BFC), Reg)
     289          99 :           .addReg(Reg, RegState::Kill)
     290         198 :           .addImm(~AlignMask)
     291         297 :           .add(predOps(ARMCC::AL));
     292          16 :     } else if (AlignMask <= 255) {
     293          42 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::BICri), Reg)
     294          14 :           .addReg(Reg, RegState::Kill)
     295          28 :           .addImm(AlignMask)
     296          56 :           .add(predOps(ARMCC::AL))
     297          28 :           .add(condCodeOp());
     298             :     } else {
     299             :       assert(!MustBeSingleInstruction &&
     300             :              "Shouldn't call emitAligningInstructions demanding a single "
     301             :              "instruction to be emitted for large stack alignment for a target "
     302             :              "without BFC.");
     303           6 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::MOVsi), Reg)
     304           2 :           .addReg(Reg, RegState::Kill)
     305           4 :           .addImm(ARM_AM::getSORegOpc(ARM_AM::lsr, NrBitsToZero))
     306           8 :           .add(predOps(ARMCC::AL))
     307           4 :           .add(condCodeOp());
     308           6 :       BuildMI(MBB, MBBI, DL, TII.get(ARM::MOVsi), Reg)
     309           2 :           .addReg(Reg, RegState::Kill)
     310           4 :           .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, NrBitsToZero))
     311           8 :           .add(predOps(ARMCC::AL))
     312           4 :           .add(condCodeOp());
     313             :     }
     314             :   } else {
     315             :     // Since this is only reached for Thumb-2 targets, the BFC instruction
     316             :     // should always be available.
     317             :     assert(CanUseBFC);
     318         267 :     BuildMI(MBB, MBBI, DL, TII.get(ARM::t2BFC), Reg)
     319          89 :         .addReg(Reg, RegState::Kill)
     320         178 :         .addImm(~AlignMask)
     321         267 :         .add(predOps(ARMCC::AL));
     322             :   }
     323         204 : }
     324             : 
     325             : /// We need the offset of the frame pointer relative to other MachineFrameInfo
     326             : /// offsets which are encoded relative to SP at function begin.
     327             : /// See also emitPrologue() for how the FP is set up.
     328             : /// Unfortunately we cannot determine this value in determineCalleeSaves() yet
     329             : /// as assignCalleeSavedSpillSlots() hasn't run at this point. Instead we use
     330             : /// this to produce a conservative estimate that we check in an assert() later.
     331             : static int getMaxFPOffset(const Function &F, const ARMFunctionInfo &AFI) {
     332             :   // This is a conservative estimation: Assume the frame pointer being r7 and
     333             :   // pc("r15") up to r8 getting spilled before (= 8 registers).
     334       12002 :   return -AFI.getArgRegsSaveSize() - (8 * 4);
     335             : }
     336             : 
     337       11064 : void ARMFrameLowering::emitPrologue(MachineFunction &MF,
     338             :                                     MachineBasicBlock &MBB) const {
     339       11064 :   MachineBasicBlock::iterator MBBI = MBB.begin();
     340       11064 :   MachineFrameInfo  &MFI = MF.getFrameInfo();
     341       11064 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     342       11064 :   MachineModuleInfo &MMI = MF.getMMI();
     343       11064 :   MCContext &Context = MMI.getContext();
     344       11064 :   const TargetMachine &TM = MF.getTarget();
     345       11064 :   const MCRegisterInfo *MRI = Context.getRegisterInfo();
     346       11064 :   const ARMBaseRegisterInfo *RegInfo = STI.getRegisterInfo();
     347       11064 :   const ARMBaseInstrInfo &TII = *STI.getInstrInfo();
     348             :   assert(!AFI->isThumb1OnlyFunction() &&
     349             :          "This emitPrologue does not support Thumb1!");
     350       11064 :   bool isARM = !AFI->isThumbFunction();
     351       11064 :   unsigned Align = STI.getFrameLowering()->getStackAlignment();
     352       11064 :   unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
     353       11064 :   unsigned NumBytes = MFI.getStackSize();
     354       11064 :   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
     355             : 
     356             :   // Debug location must be unknown since the first debug location is used
     357             :   // to determine the end of the prologue.
     358       14358 :   DebugLoc dl;
     359             :   
     360       11064 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
     361             : 
     362             :   // Determine the sizes of each callee-save spill areas and record which frame
     363             :   // belongs to which callee-save spill areas.
     364       11064 :   unsigned GPRCS1Size = 0, GPRCS2Size = 0, DPRCSSize = 0;
     365       11064 :   int FramePtrSpillFI = 0;
     366       11064 :   int D8SpillFI = 0;
     367             : 
     368             :   // All calls are tail calls in GHC calling conv, and functions have no
     369             :   // prologue/epilogue.
     370       22128 :   if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
     371        7770 :     return;
     372             : 
     373       14356 :   StackAdjustingInsts DefCFAOffsetCandidates;
     374       11062 :   bool HasFP = hasFP(MF);
     375             : 
     376             :   // Allocate the vararg register save area.
     377       11062 :   if (ArgRegsSaveSize) {
     378          78 :     emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize,
     379             :                  MachineInstr::FrameSetup);
     380          39 :     DefCFAOffsetCandidates.addInst(std::prev(MBBI), ArgRegsSaveSize, true);
     381             :   }
     382             : 
     383       18830 :   if (!AFI->hasStackFrame() &&
     384       15594 :       (!STI.isTargetWindows() || !WindowsRequiresStackProbe(MF, NumBytes))) {
     385        7768 :     if (NumBytes - ArgRegsSaveSize != 0) {
     386         616 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -(NumBytes - ArgRegsSaveSize),
     387             :                    MachineInstr::FrameSetup);
     388         616 :       DefCFAOffsetCandidates.addInst(std::prev(MBBI),
     389             :                                      NumBytes - ArgRegsSaveSize, true);
     390             :     }
     391        7768 :     DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
     392             :     return;
     393             :   }
     394             : 
     395             :   // Determine spill area sizes.
     396       30954 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
     397       21072 :     unsigned Reg = CSI[i].getReg();
     398       10536 :     int FI = CSI[i].getFrameIdx();
     399       10536 :     switch (Reg) {
     400        1961 :     case ARM::R8:
     401             :     case ARM::R9:
     402             :     case ARM::R10:
     403             :     case ARM::R11:
     404             :     case ARM::R12:
     405        1961 :       if (STI.splitFramePushPop(MF)) {
     406         210 :         GPRCS2Size += 4;
     407         210 :         break;
     408             :       }
     409             :       LLVM_FALLTHROUGH;
     410             :     case ARM::R0:
     411             :     case ARM::R1:
     412             :     case ARM::R2:
     413             :     case ARM::R3:
     414             :     case ARM::R4:
     415             :     case ARM::R5:
     416             :     case ARM::R6:
     417             :     case ARM::R7:
     418             :     case ARM::LR:
     419        9154 :       if (Reg == FramePtr)
     420         952 :         FramePtrSpillFI = FI;
     421        9154 :       GPRCS1Size += 4;
     422        9154 :       break;
     423        1172 :     default:
     424             :       // This is a DPR. Exclude the aligned DPRCS2 spills.
     425        1172 :       if (Reg == ARM::D8)
     426         201 :         D8SpillFI = FI;
     427        1172 :       if (Reg < ARM::D8 || Reg >= ARM::D8 + AFI->getNumAlignedDPRCS2Regs())
     428         905 :         DPRCSSize += 8;
     429             :     }
     430             :   }
     431             : 
     432             :   // Move past area 1.
     433        9882 :   MachineBasicBlock::iterator LastPush = MBB.end(), GPRCS1Push, GPRCS2Push;
     434        3294 :   if (GPRCS1Size > 0) {
     435        6574 :     GPRCS1Push = LastPush = MBBI++;
     436             :     DefCFAOffsetCandidates.addInst(LastPush, GPRCS1Size, true);
     437             :   }
     438             : 
     439             :   // Determine starting offsets of spill areas.
     440        3294 :   unsigned GPRCS1Offset = NumBytes - ArgRegsSaveSize - GPRCS1Size;
     441        3294 :   unsigned GPRCS2Offset = GPRCS1Offset - GPRCS2Size;
     442        3473 :   unsigned DPRAlign = DPRCSSize ? std::min(8U, Align) : 4U;
     443        3294 :   unsigned DPRGapSize = (GPRCS1Size + GPRCS2Size + ArgRegsSaveSize) % DPRAlign;
     444        3294 :   unsigned DPRCSOffset = GPRCS2Offset - DPRGapSize - DPRCSSize;
     445        3294 :   int FramePtrOffsetInPush = 0;
     446        3294 :   if (HasFP) {
     447         952 :     int FPOffset = MFI.getObjectOffset(FramePtrSpillFI);
     448             :     assert(getMaxFPOffset(*MF.getFunction(), *AFI) <= FPOffset &&
     449             :            "Max FP estimation is wrong");
     450         952 :     FramePtrOffsetInPush = FPOffset + ArgRegsSaveSize;
     451             :     AFI->setFramePtrSpillOffset(MFI.getObjectOffset(FramePtrSpillFI) +
     452             :                                 NumBytes);
     453             :   }
     454        6588 :   AFI->setGPRCalleeSavedArea1Offset(GPRCS1Offset);
     455        6588 :   AFI->setGPRCalleeSavedArea2Offset(GPRCS2Offset);
     456        6588 :   AFI->setDPRCalleeSavedAreaOffset(DPRCSOffset);
     457             : 
     458             :   // Move past area 2.
     459        3294 :   if (GPRCS2Size > 0) {
     460         230 :     GPRCS2Push = LastPush = MBBI++;
     461             :     DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size);
     462             :   }
     463             : 
     464             :   // Prolog/epilog inserter assumes we correctly align DPRs on the stack, so our
     465             :   // .cfi_offset operations will reflect that.
     466        3294 :   if (DPRGapSize) {
     467             :     assert(DPRGapSize == 4 && "unexpected alignment requirements for DPRs");
     468          32 :     if (LastPush != MBB.end() &&
     469          10 :         tryFoldSPUpdateIntoPushPop(STI, MF, &*LastPush, DPRGapSize))
     470             :       DefCFAOffsetCandidates.addExtraBytes(LastPush, DPRGapSize);
     471             :     else {
     472          18 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -DPRGapSize,
     473             :                    MachineInstr::FrameSetup);
     474           9 :       DefCFAOffsetCandidates.addInst(std::prev(MBBI), DPRGapSize);
     475             :     }
     476             :   }
     477             : 
     478             :   // Move past area 3.
     479        3294 :   if (DPRCSSize > 0) {
     480             :     // Since vpush register list cannot have gaps, there may be multiple vpush
     481             :     // instructions in the prologue.
     482         970 :     while (MBBI->getOpcode() == ARM::VSTMDDB_UPD) {
     483         408 :       DefCFAOffsetCandidates.addInst(MBBI, sizeOfSPAdjustment(*MBBI));
     484         408 :       LastPush = MBBI++;
     485             :     }
     486             :   }
     487             : 
     488             :   // Move past the aligned DPRCS2 area.
     489        3294 :   if (AFI->getNumAlignedDPRCS2Regs() > 0) {
     490          39 :     MBBI = skipAlignedDPRCS2Spills(MBBI, AFI->getNumAlignedDPRCS2Regs());
     491             :     // The code inserted by emitAlignedDPRCS2Spills realigns the stack, and
     492             :     // leaves the stack pointer pointing to the DPRCS2 area.
     493             :     //
     494             :     // Adjust NumBytes to represent the stack slots below the DPRCS2 area.
     495          39 :     NumBytes += MFI.getObjectOffset(D8SpillFI);
     496             :   } else
     497             :     NumBytes = DPRCSOffset;
     498             : 
     499        6588 :   if (STI.isTargetWindows() && WindowsRequiresStackProbe(MF, NumBytes)) {
     500           3 :     uint32_t NumWords = NumBytes >> 2;
     501             : 
     502           3 :     if (NumWords < 65536)
     503           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi16), ARM::R4)
     504           6 :           .addImm(NumWords)
     505           3 :           .setMIFlags(MachineInstr::FrameSetup)
     506           9 :           .add(predOps(ARMCC::AL));
     507             :     else
     508           0 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi32imm), ARM::R4)
     509           0 :         .addImm(NumWords)
     510           0 :         .setMIFlags(MachineInstr::FrameSetup);
     511             : 
     512           3 :     switch (TM.getCodeModel()) {
     513           1 :     case CodeModel::Small:
     514             :     case CodeModel::Medium:
     515             :     case CodeModel::Kernel:
     516           3 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tBL))
     517           4 :           .add(predOps(ARMCC::AL))
     518           1 :           .addExternalSymbol("__chkstk")
     519           1 :           .addReg(ARM::R4, RegState::Implicit)
     520           1 :           .setMIFlags(MachineInstr::FrameSetup);
     521           1 :       break;
     522           2 :     case CodeModel::Large:
     523           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::t2MOVi32imm), ARM::R12)
     524           2 :         .addExternalSymbol("__chkstk")
     525           2 :         .setMIFlags(MachineInstr::FrameSetup);
     526             : 
     527           6 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tBLXr))
     528           8 :           .add(predOps(ARMCC::AL))
     529           2 :           .addReg(ARM::R12, RegState::Kill)
     530           2 :           .addReg(ARM::R4, RegState::Implicit)
     531           2 :           .setMIFlags(MachineInstr::FrameSetup);
     532           2 :       break;
     533             :     }
     534             : 
     535           9 :     BuildMI(MBB, MBBI, dl, TII.get(ARM::t2SUBrr), ARM::SP)
     536           3 :         .addReg(ARM::SP, RegState::Kill)
     537           3 :         .addReg(ARM::R4, RegState::Kill)
     538           3 :         .setMIFlags(MachineInstr::FrameSetup)
     539          12 :         .add(predOps(ARMCC::AL))
     540           6 :         .add(condCodeOp());
     541           3 :     NumBytes = 0;
     542             :   }
     543             : 
     544        3291 :   if (NumBytes) {
     545             :     // Adjust SP after all the callee-save spills.
     546        2102 :     if (AFI->getNumAlignedDPRCS2Regs() == 0 &&
     547        1036 :         tryFoldSPUpdateIntoPushPop(STI, MF, &*LastPush, NumBytes))
     548             :       DefCFAOffsetCandidates.addExtraBytes(LastPush, NumBytes);
     549             :     else {
     550        2092 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, -NumBytes,
     551             :                    MachineInstr::FrameSetup);
     552        1046 :       DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes);
     553             :     }
     554             : 
     555        1066 :     if (HasFP && isARM)
     556             :       // Restore from fp only in ARM mode: e.g. sub sp, r7, #24
     557             :       // Note it's not safe to do this in Thumb2 mode because it would have
     558             :       // taken two instructions:
     559             :       // mov sp, r7
     560             :       // sub sp, #24
     561             :       // If an interrupt is taken between the two instructions, then sp is in
     562             :       // an inconsistent state (pointing to the middle of callee-saved area).
     563             :       // The interrupt handler can end up clobbering the registers.
     564             :       AFI->setShouldRestoreSPFromFP(true);
     565             :   }
     566             : 
     567             :   // Set FP to point to the stack slot that contains the previous FP.
     568             :   // For iOS, FP is R7, which has now been stored in spill area 1.
     569             :   // Otherwise, if this is not iOS, all the callee-saved registers go
     570             :   // into spill area 1, including the FP in R11.  In either case, it
     571             :   // is in area one and the adjustment needs to take place just after
     572             :   // that push.
     573        3294 :   if (HasFP) {
     574         952 :     MachineBasicBlock::iterator AfterPush = std::next(GPRCS1Push);
     575         952 :     unsigned PushSize = sizeOfSPAdjustment(*GPRCS1Push);
     576         952 :     emitRegPlusImmediate(!AFI->isThumbFunction(), MBB, AfterPush,
     577             :                          dl, TII, FramePtr, ARM::SP,
     578         952 :                          PushSize + FramePtrOffsetInPush,
     579             :                          MachineInstr::FrameSetup);
     580         952 :     if (FramePtrOffsetInPush + PushSize != 0) {
     581        1888 :       unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
     582         472 :           nullptr, MRI->getDwarfRegNum(FramePtr, true),
     583         944 :           -(ArgRegsSaveSize - FramePtrOffsetInPush)));
     584        1416 :       BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     585         472 :           .addCFIIndex(CFIIndex)
     586         472 :           .setMIFlags(MachineInstr::FrameSetup);
     587             :     } else {
     588             :       unsigned CFIIndex =
     589        1440 :           MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
     590         960 :               nullptr, MRI->getDwarfRegNum(FramePtr, true)));
     591        1440 :       BuildMI(MBB, AfterPush, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     592         480 :           .addCFIIndex(CFIIndex)
     593         480 :           .setMIFlags(MachineInstr::FrameSetup);
     594             :     }
     595             :   }
     596             : 
     597             :   // Now that the prologue's actual instructions are finalised, we can insert
     598             :   // the necessary DWARF cf instructions to describe the situation. Start by
     599             :   // recording where each register ended up:
     600        3294 :   if (GPRCS1Size > 0) {
     601        3287 :     MachineBasicBlock::iterator Pos = std::next(GPRCS1Push);
     602             :     int CFIIndex;
     603       23645 :     for (const auto &Entry : CSI) {
     604       10497 :       unsigned Reg = Entry.getReg();
     605       10497 :       int FI = Entry.getFrameIdx();
     606       10497 :       switch (Reg) {
     607        1961 :       case ARM::R8:
     608             :       case ARM::R9:
     609             :       case ARM::R10:
     610             :       case ARM::R11:
     611             :       case ARM::R12:
     612        1961 :         if (STI.splitFramePushPop(MF))
     613             :           break;
     614             :         LLVM_FALLTHROUGH;
     615             :       case ARM::R0:
     616             :       case ARM::R1:
     617             :       case ARM::R2:
     618             :       case ARM::R3:
     619             :       case ARM::R4:
     620             :       case ARM::R5:
     621             :       case ARM::R6:
     622             :       case ARM::R7:
     623             :       case ARM::LR:
     624       45770 :         CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
     625       18308 :             nullptr, MRI->getDwarfRegNum(Reg, true), MFI.getObjectOffset(FI)));
     626       27462 :         BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     627       18308 :             .addCFIIndex(CFIIndex)
     628        9154 :             .setMIFlags(MachineInstr::FrameSetup);
     629        9154 :         break;
     630             :       }
     631             :     }
     632             :   }
     633             : 
     634        3294 :   if (GPRCS2Size > 0) {
     635         115 :     MachineBasicBlock::iterator Pos = std::next(GPRCS2Push);
     636        1564 :     for (const auto &Entry : CSI) {
     637        1104 :       unsigned Reg = Entry.getReg();
     638        1104 :       int FI = Entry.getFrameIdx();
     639        1104 :       switch (Reg) {
     640         210 :       case ARM::R8:
     641             :       case ARM::R9:
     642             :       case ARM::R10:
     643             :       case ARM::R11:
     644             :       case ARM::R12:
     645         210 :         if (STI.splitFramePushPop(MF)) {
     646         210 :           unsigned DwarfReg =  MRI->getDwarfRegNum(Reg, true);
     647         210 :           unsigned Offset = MFI.getObjectOffset(FI);
     648             :           unsigned CFIIndex = MF.addFrameInst(
     649         840 :               MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     650         630 :           BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     651         210 :               .addCFIIndex(CFIIndex)
     652         210 :               .setMIFlags(MachineInstr::FrameSetup);
     653             :         }
     654             :         break;
     655             :       }
     656             :     }
     657             :   }
     658             : 
     659        3294 :   if (DPRCSSize > 0) {
     660             :     // Since vpush register list cannot have gaps, there may be multiple vpush
     661             :     // instructions in the prologue.
     662         179 :     MachineBasicBlock::iterator Pos = std::next(LastPush);
     663        2215 :     for (const auto &Entry : CSI) {
     664        1499 :       unsigned Reg = Entry.getReg();
     665        1499 :       int FI = Entry.getFrameIdx();
     666        1499 :       if ((Reg >= ARM::D0 && Reg <= ARM::D31) &&
     667         780 :           (Reg < ARM::D8 || Reg >= ARM::D8 + AFI->getNumAlignedDPRCS2Regs())) {
     668         905 :         unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
     669         905 :         unsigned Offset = MFI.getObjectOffset(FI);
     670             :         unsigned CFIIndex = MF.addFrameInst(
     671        3620 :             MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
     672        2715 :         BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION))
     673         905 :             .addCFIIndex(CFIIndex)
     674         905 :             .setMIFlags(MachineInstr::FrameSetup);
     675             :       }
     676             :     }
     677             :   }
     678             : 
     679             :   // Now we can emit descriptions of where the canonical frame address was
     680             :   // throughout the process. If we have a frame pointer, it takes over the job
     681             :   // half-way through, so only the first few .cfi_def_cfa_offset instructions
     682             :   // actually get emitted.
     683        3294 :   DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP);
     684             : 
     685        6588 :   if (STI.isTargetELF() && hasFP(MF))
     686         736 :     MFI.setOffsetAdjustment(MFI.getOffsetAdjustment() -
     687         368 :                             AFI->getFramePtrSpillOffset());
     688             : 
     689        6588 :   AFI->setGPRCalleeSavedArea1Size(GPRCS1Size);
     690        6588 :   AFI->setGPRCalleeSavedArea2Size(GPRCS2Size);
     691        6588 :   AFI->setDPRCalleeSavedGapSize(DPRGapSize);
     692        6588 :   AFI->setDPRCalleeSavedAreaSize(DPRCSSize);
     693             : 
     694             :   // If we need dynamic stack realignment, do it here. Be paranoid and make
     695             :   // sure if we also have VLAs, we have a base pointer for frame access.
     696             :   // If aligned NEON registers were spilled, the stack has already been
     697             :   // realigned.
     698        3294 :   if (!AFI->getNumAlignedDPRCS2Regs() && RegInfo->needsStackRealignment(MF)) {
     699         165 :     unsigned MaxAlign = MFI.getMaxAlignment();
     700             :     assert(!AFI->isThumb1OnlyFunction());
     701         165 :     if (!AFI->isThumbFunction()) {
     702          99 :       emitAligningInstructions(MF, AFI, TII, MBB, MBBI, dl, ARM::SP, MaxAlign,
     703             :                                false);
     704             :     } else {
     705             :       // We cannot use sp as source/dest register here, thus we're using r4 to
     706             :       // perform the calculations. We're emitting the following sequence:
     707             :       // mov r4, sp
     708             :       // -- use emitAligningInstructions to produce best sequence to zero
     709             :       // -- out lower bits in r4
     710             :       // mov sp, r4
     711             :       // FIXME: It will be better just to find spare register here.
     712         198 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::R4)
     713          66 :           .addReg(ARM::SP, RegState::Kill)
     714         198 :           .add(predOps(ARMCC::AL));
     715          66 :       emitAligningInstructions(MF, AFI, TII, MBB, MBBI, dl, ARM::R4, MaxAlign,
     716             :                                false);
     717         198 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     718          66 :           .addReg(ARM::R4, RegState::Kill)
     719         198 :           .add(predOps(ARMCC::AL));
     720             :     }
     721             : 
     722             :     AFI->setShouldRestoreSPFromFP(true);
     723             :   }
     724             : 
     725             :   // If we need a base pointer, set it up here. It's whatever the value
     726             :   // of the stack pointer is at this point. Any variable size objects
     727             :   // will be allocated after this, so we can still use the base pointer
     728             :   // to reference locals.
     729             :   // FIXME: Clarify FrameSetup flags here.
     730        3294 :   if (RegInfo->hasBasePointer(MF)) {
     731           6 :     if (isARM)
     732           9 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), RegInfo->getBaseRegister())
     733           3 :           .addReg(ARM::SP)
     734          12 :           .add(predOps(ARMCC::AL))
     735           6 :           .add(condCodeOp());
     736             :     else
     737           9 :       BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), RegInfo->getBaseRegister())
     738           3 :           .addReg(ARM::SP)
     739           9 :           .add(predOps(ARMCC::AL));
     740             :   }
     741             : 
     742             :   // If the frame has variable sized objects then the epilogue must restore
     743             :   // the sp from fp. We can assume there's an FP here since hasFP already
     744             :   // checks for hasVarSizedObjects.
     745        3294 :   if (MFI.hasVarSizedObjects())
     746             :     AFI->setShouldRestoreSPFromFP(true);
     747             : }
     748             : 
     749       11071 : void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
     750             :                                     MachineBasicBlock &MBB) const {
     751       11071 :   MachineFrameInfo &MFI = MF.getFrameInfo();
     752       11071 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     753       11071 :   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
     754             :   const ARMBaseInstrInfo &TII =
     755       11071 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
     756             :   assert(!AFI->isThumb1OnlyFunction() &&
     757             :          "This emitEpilogue does not support Thumb1!");
     758       11071 :   bool isARM = !AFI->isThumbFunction();
     759             : 
     760       11071 :   unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
     761       11071 :   int NumBytes = (int)MFI.getStackSize();
     762       11071 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
     763             : 
     764             :   // All calls are tail calls in GHC calling conv, and functions have no
     765             :   // prologue/epilogue.
     766       22142 :   if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
     767           2 :     return;
     768             : 
     769             :   // First put ourselves on the first (from top) terminator instructions.
     770       11069 :   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
     771       44276 :   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
     772             : 
     773       11069 :   if (!AFI->hasStackFrame()) {
     774        7830 :     if (NumBytes - ArgRegsSaveSize != 0)
     775         304 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ArgRegsSaveSize);
     776             :   } else {
     777             :     // Unwind MBBI to point to first LDR / VLDRD.
     778        3239 :     const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
     779        6478 :     if (MBBI != MBB.begin()) {
     780             :       do {
     781        4312 :         --MBBI;
     782       12541 :       } while (MBBI != MBB.begin() && isCSRestore(*MBBI, TII, CSRegs));
     783        3105 :       if (!isCSRestore(*MBBI, TII, CSRegs))
     784             :         ++MBBI;
     785             :     }
     786             : 
     787             :     // Move SP to start of FP callee save spill area.
     788        6478 :     NumBytes -= (ArgRegsSaveSize +
     789        6478 :                  AFI->getGPRCalleeSavedArea1Size() +
     790        6478 :                  AFI->getGPRCalleeSavedArea2Size() +
     791        6478 :                  AFI->getDPRCalleeSavedGapSize() +
     792        3239 :                  AFI->getDPRCalleeSavedAreaSize());
     793             : 
     794             :     // Reset SP based on frame pointer only if the stack frame extends beyond
     795             :     // frame pointer stack slot or target is ELF and the function has FP.
     796        3239 :     if (AFI->shouldRestoreSPFromFP()) {
     797         377 :       NumBytes = AFI->getFramePtrSpillOffset() - NumBytes;
     798         377 :       if (NumBytes) {
     799         249 :         if (isARM)
     800         153 :           emitARMRegPlusImmediate(MBB, MBBI, dl, ARM::SP, FramePtr, -NumBytes,
     801             :                                   ARMCC::AL, 0, TII);
     802             :         else {
     803             :           // It's not possible to restore SP from FP in a single instruction.
     804             :           // For iOS, this looks like:
     805             :           // mov sp, r7
     806             :           // sub sp, #24
     807             :           // This is bad, if an interrupt is taken after the mov, sp is in an
     808             :           // inconsistent state.
     809             :           // Use the first callee-saved register as a scratch register.
     810             :           assert(!MFI.getPristineRegs(MF).test(ARM::R4) &&
     811             :                  "No scratch register to restore SP from FP!");
     812          96 :           emitT2RegPlusImmediate(MBB, MBBI, dl, ARM::R4, FramePtr, -NumBytes,
     813             :                                  ARMCC::AL, 0, TII);
     814         288 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     815          96 :               .addReg(ARM::R4)
     816         288 :               .add(predOps(ARMCC::AL));
     817             :         }
     818             :       } else {
     819             :         // Thumb2 or ARM.
     820         128 :         if (isARM)
     821         384 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), ARM::SP)
     822         128 :               .addReg(FramePtr)
     823         512 :               .add(predOps(ARMCC::AL))
     824         256 :               .add(condCodeOp());
     825             :         else
     826           0 :           BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr), ARM::SP)
     827           0 :               .addReg(FramePtr)
     828           0 :               .add(predOps(ARMCC::AL));
     829             :       }
     830        3554 :     } else if (NumBytes &&
     831        1384 :                !tryFoldSPUpdateIntoPushPop(STI, MF, &*MBBI, NumBytes))
     832         680 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes);
     833             : 
     834             :     // Increment past our save areas.
     835        6478 :     if (MBBI != MBB.end() && AFI->getDPRCalleeSavedAreaSize()) {
     836         190 :       MBBI++;
     837             :       // Since vpop register list cannot have gaps, there may be multiple vpop
     838             :       // instructions in the epilogue.
     839         850 :       while (MBBI != MBB.end() && MBBI->getOpcode() == ARM::VLDMDIA_UPD)
     840          23 :         MBBI++;
     841             :     }
     842        3239 :     if (AFI->getDPRCalleeSavedGapSize()) {
     843             :       assert(AFI->getDPRCalleeSavedGapSize() == 4 &&
     844             :              "unexpected DPR alignment gap");
     845          10 :       emitSPUpdate(isARM, MBB, MBBI, dl, TII, AFI->getDPRCalleeSavedGapSize());
     846             :     }
     847             : 
     848        3239 :     if (AFI->getGPRCalleeSavedArea2Size()) MBBI++;
     849        3239 :     if (AFI->getGPRCalleeSavedArea1Size()) MBBI++;
     850             :   }
     851             : 
     852       11069 :   if (ArgRegsSaveSize)
     853          39 :     emitSPUpdate(isARM, MBB, MBBI, dl, TII, ArgRegsSaveSize);
     854             : }
     855             : 
     856             : /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
     857             : /// debug info.  It's the same as what we use for resolving the code-gen
     858             : /// references for now.  FIXME: This can go wrong when references are
     859             : /// SP-relative and simple call frames aren't used.
     860             : int
     861          22 : ARMFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
     862             :                                          unsigned &FrameReg) const {
     863          22 :   return ResolveFrameIndexReference(MF, FI, FrameReg, 0);
     864             : }
     865             : 
     866             : int
     867       12875 : ARMFrameLowering::ResolveFrameIndexReference(const MachineFunction &MF,
     868             :                                              int FI, unsigned &FrameReg,
     869             :                                              int SPAdj) const {
     870       12875 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
     871             :   const ARMBaseRegisterInfo *RegInfo = static_cast<const ARMBaseRegisterInfo *>(
     872       12875 :       MF.getSubtarget().getRegisterInfo());
     873       12875 :   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
     874       12875 :   int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
     875       12875 :   int FPOffset = Offset - AFI->getFramePtrSpillOffset();
     876       25750 :   bool isFixed = MFI.isFixedObjectIndex(FI);
     877             : 
     878       12875 :   FrameReg = ARM::SP;
     879       12875 :   Offset += SPAdj;
     880             : 
     881             :   // SP can move around if there are allocas.  We may also lose track of SP
     882             :   // when emergency spilling inside a non-reserved call frame setup.
     883       12875 :   bool hasMovingSP = !hasReservedCallFrame(MF);
     884             : 
     885             :   // When dynamically realigning the stack, use the frame pointer for
     886             :   // parameters, and the stack/base pointer for locals.
     887       12875 :   if (RegInfo->needsStackRealignment(MF)) {
     888             :     assert(hasFP(MF) && "dynamic stack realignment without a FP!");
     889        6926 :     if (isFixed) {
     890          65 :       FrameReg = RegInfo->getFrameRegister(MF);
     891          65 :       Offset = FPOffset;
     892        6861 :     } else if (hasMovingSP) {
     893             :       assert(RegInfo->hasBasePointer(MF) &&
     894             :              "VLAs and dynamic stack alignment, but missing base pointer!");
     895         449 :       FrameReg = RegInfo->getBaseRegister();
     896             :     }
     897             :     return Offset;
     898             :   }
     899             : 
     900             :   // If there is a frame pointer, use it when we can.
     901        5949 :   if (hasFP(MF) && AFI->hasStackFrame()) {
     902             :     // Use frame pointer to reference fixed objects. Use it for locals if
     903             :     // there are VLAs (and thus the SP isn't reliable as a base).
     904        1417 :     if (isFixed || (hasMovingSP && !RegInfo->hasBasePointer(MF))) {
     905         217 :       FrameReg = RegInfo->getFrameRegister(MF);
     906         217 :       return FPOffset;
     907        1200 :     } else if (hasMovingSP) {
     908             :       assert(RegInfo->hasBasePointer(MF) && "missing base pointer!");
     909             :       if (AFI->isThumb2Function()) {
     910             :         // Try to use the frame pointer if we can, else use the base pointer
     911             :         // since it's available. This is handy for the emergency spill slot, in
     912             :         // particular.
     913           0 :         if (FPOffset >= -255 && FPOffset < 0) {
     914           0 :           FrameReg = RegInfo->getFrameRegister(MF);
     915           0 :           return FPOffset;
     916             :         }
     917             :       }
     918             :     } else if (AFI->isThumb2Function()) {
     919             :       // Use  add <rd>, sp, #<imm8>
     920             :       //      ldr <rd>, [sp, #<imm8>]
     921             :       // if at all possible to save space.
     922         361 :       if (Offset >= 0 && (Offset & 3) == 0 && Offset <= 1020)
     923             :         return Offset;
     924             :       // In Thumb2 mode, the negative offset is very limited. Try to avoid
     925             :       // out of range references. ldr <rt>,[<rn>, #-<imm8>]
     926          24 :       if (FPOffset >= -255 && FPOffset < 0) {
     927          24 :         FrameReg = RegInfo->getFrameRegister(MF);
     928          24 :         return FPOffset;
     929             :       }
     930         839 :     } else if (Offset > (FPOffset < 0 ? -FPOffset : FPOffset)) {
     931             :       // Otherwise, use SP or FP, whichever is closer to the stack slot.
     932         254 :       FrameReg = RegInfo->getFrameRegister(MF);
     933         254 :       return FPOffset;
     934             :     }
     935             :   }
     936             :   // Use the base pointer if we have one.
     937        5117 :   if (RegInfo->hasBasePointer(MF))
     938           0 :     FrameReg = RegInfo->getBaseRegister();
     939             :   return Offset;
     940             : }
     941             : 
     942        9882 : void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
     943             :                                     MachineBasicBlock::iterator MI,
     944             :                                     const std::vector<CalleeSavedInfo> &CSI,
     945             :                                     unsigned StmOpc, unsigned StrOpc,
     946             :                                     bool NoGap,
     947             :                                     bool(*Func)(unsigned, bool),
     948             :                                     unsigned NumAlignedDPRCS2Regs,
     949             :                                     unsigned MIFlags) const {
     950        9882 :   MachineFunction &MF = *MBB.getParent();
     951        9882 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
     952        9882 :   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
     953             : 
     954       19764 :   DebugLoc DL;
     955             : 
     956             :   typedef std::pair<unsigned, bool> RegAndKill;
     957       19764 :   SmallVector<RegAndKill, 4> Regs;
     958       19764 :   unsigned i = CSI.size();
     959       19789 :   while (i != 0) {
     960             :     unsigned LastReg = 0;
     961       41515 :     for (; i != 0; --i) {
     962       63266 :       unsigned Reg = CSI[i-1].getReg();
     963       31633 :       if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue;
     964             : 
     965             :       // D-registers in the aligned area DPRCS2 are NOT spilled here.
     966       10561 :       if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs)
     967         267 :         continue;
     968             : 
     969       10294 :       const MachineRegisterInfo &MRI = MF.getRegInfo();
     970       10294 :       bool isLiveIn = MRI.isLiveIn(Reg);
     971       20567 :       if (!isLiveIn && !MRI.isReserved(Reg))
     972        9315 :         MBB.addLiveIn(Reg);
     973             :       // If NoGap is true, push consecutive registers and then leave the rest
     974             :       // for other instructions. e.g.
     975             :       // vpush {d8, d10, d11} -> vpush {d8}, vpush {d10, d11}
     976       10294 :       if (NoGap && LastReg && LastReg != Reg-1)
     977             :         break;
     978       10269 :       LastReg = Reg;
     979             :       // Do not set a kill flag on values that are also marked as live-in. This
     980             :       // happens with the @llvm-returnaddress intrinsic and with arguments
     981             :       // passed in callee saved registers.
     982             :       // Omitting the kill flags is conservatively correct even if the live-in
     983             :       // is not used after all.
     984       20538 :       Regs.push_back(std::make_pair(Reg, /*isKill=*/!isLiveIn));
     985             :     }
     986             : 
     987        9907 :     if (Regs.empty())
     988        6301 :       continue;
     989             : 
     990        7212 :     std::sort(Regs.begin(), Regs.end(), [&](const RegAndKill &LHS,
     991             :                                             const RegAndKill &RHS) {
     992       40794 :       return TRI.getEncodingValue(LHS.first) < TRI.getEncodingValue(RHS.first);
     993             :     });
     994             : 
     995        3606 :     if (Regs.size() > 1 || StrOpc== 0) {
     996        9990 :       MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StmOpc), ARM::SP)
     997        3330 :                                     .addReg(ARM::SP)
     998        3330 :                                     .setMIFlags(MIFlags)
     999        9990 :                                     .add(predOps(ARMCC::AL));
    1000       16653 :       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
    1001       39972 :         MIB.addReg(Regs[i].first, getKillRegState(Regs[i].second));
    1002         276 :     } else if (Regs.size() == 1) {
    1003         828 :       BuildMI(MBB, MI, DL, TII.get(StrOpc), ARM::SP)
    1004         828 :           .addReg(Regs[0].first, getKillRegState(Regs[0].second))
    1005         276 :           .addReg(ARM::SP)
    1006         276 :           .setMIFlags(MIFlags)
    1007         276 :           .addImm(-4)
    1008         828 :           .add(predOps(ARMCC::AL));
    1009             :     }
    1010        3606 :     Regs.clear();
    1011             : 
    1012             :     // Put any subsequent vpush instructions before this one: they will refer to
    1013             :     // higher register numbers so need to be pushed first in order to preserve
    1014             :     // monotonicity.
    1015        7212 :     if (MI != MBB.begin())
    1016             :       --MI;
    1017             :   }
    1018        9882 : }
    1019             : 
    1020        9717 : void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB,
    1021             :                                    MachineBasicBlock::iterator MI,
    1022             :                                    std::vector<CalleeSavedInfo> &CSI,
    1023             :                                    unsigned LdmOpc, unsigned LdrOpc,
    1024             :                                    bool isVarArg, bool NoGap,
    1025             :                                    bool(*Func)(unsigned, bool),
    1026             :                                    unsigned NumAlignedDPRCS2Regs) const {
    1027        9717 :   MachineFunction &MF = *MBB.getParent();
    1028        9717 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1029        9717 :   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
    1030        9717 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1031       19434 :   DebugLoc DL;
    1032        9717 :   bool isTailCall = false;
    1033        9717 :   bool isInterrupt = false;
    1034        9717 :   bool isTrap = false;
    1035       19434 :   if (MBB.end() != MI) {
    1036       19278 :     DL = MI->getDebugLoc();
    1037       19278 :     unsigned RetOpcode = MI->getOpcode();
    1038        9639 :     isTailCall = (RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNri);
    1039        9639 :     isInterrupt =
    1040        9639 :         RetOpcode == ARM::SUBS_PC_LR || RetOpcode == ARM::t2SUBS_PC_LR;
    1041        9639 :     isTrap =
    1042        9639 :         RetOpcode == ARM::TRAP || RetOpcode == ARM::TRAPNaCl ||
    1043        9639 :         RetOpcode == ARM::tTRAP;
    1044             :   }
    1045             : 
    1046       19434 :   SmallVector<unsigned, 4> Regs;
    1047       19434 :   unsigned i = CSI.size();
    1048       19457 :   while (i != 0) {
    1049             :     unsigned LastReg = 0;
    1050             :     bool DeleteRet = false;
    1051       40712 :     for (; i != 0; --i) {
    1052       61990 :       unsigned Reg = CSI[i-1].getReg();
    1053       51846 :       if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue;
    1054             : 
    1055             :       // The aligned reloads from area DPRCS2 are not inserted here.
    1056       10347 :       if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs)
    1057         203 :         continue;
    1058             : 
    1059        3203 :       if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt &&
    1060       13225 :           !isTrap && STI.hasV5TOps()) {
    1061        2447 :         if (MBB.succ_empty()) {
    1062        2411 :           Reg = ARM::PC;
    1063        2411 :           DeleteRet = true;
    1064        2411 :           LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_RET : ARM::LDMIA_RET;
    1065             :         } else
    1066          36 :           LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
    1067             :         // Fold the return instruction into the LDM.
    1068             :       }
    1069             : 
    1070             :       // If NoGap is true, pop consecutive registers and then leave the rest
    1071             :       // for other instructions. e.g.
    1072             :       // vpop {d8, d10, d11} -> vpop {d8}, vpop {d10, d11}
    1073       10144 :       if (NoGap && LastReg && LastReg != Reg-1)
    1074             :         break;
    1075             : 
    1076       10121 :       LastReg = Reg;
    1077       10121 :       Regs.push_back(Reg);
    1078             :     }
    1079             : 
    1080        9740 :     if (Regs.empty())
    1081        6181 :       continue;
    1082             : 
    1083        7118 :     std::sort(Regs.begin(), Regs.end(), [&](unsigned LHS, unsigned RHS) {
    1084       33242 :       return TRI.getEncodingValue(LHS) < TRI.getEncodingValue(RHS);
    1085             :     });
    1086             : 
    1087        3559 :     if (Regs.size() > 1 || LdrOpc == 0) {
    1088        9897 :       MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdmOpc), ARM::SP)
    1089        3299 :                                     .addReg(ARM::SP)
    1090        9897 :                                     .add(predOps(ARMCC::AL));
    1091       16459 :       for (unsigned i = 0, e = Regs.size(); i < e; ++i)
    1092       19722 :         MIB.addReg(Regs[i], getDefRegState(true));
    1093        3299 :       if (DeleteRet) {
    1094        4546 :         if (MI != MBB.end()) {
    1095        4546 :           MIB.copyImplicitOps(*MI);
    1096        2273 :           MI->eraseFromParent();
    1097             :         }
    1098             :         // If LR is not restored, mark it in CSI.
    1099        9092 :         for (CalleeSavedInfo &I : CSI) {
    1100        2273 :           if (I.getReg() != ARM::LR)
    1101             :             continue;
    1102             :           I.setRestored(false);
    1103             :           break;
    1104             :         }
    1105             :       }
    1106        6598 :       MI = MIB;
    1107         260 :     } else if (Regs.size() == 1) {
    1108             :       // If we adjusted the reg to PC from LR above, switch it back here. We
    1109             :       // only do that for LDM.
    1110         260 :       if (Regs[0] == ARM::PC)
    1111         138 :         Regs[0] = ARM::LR;
    1112             :       MachineInstrBuilder MIB =
    1113         780 :         BuildMI(MBB, MI, DL, TII.get(LdrOpc), Regs[0])
    1114         260 :           .addReg(ARM::SP, RegState::Define)
    1115         260 :           .addReg(ARM::SP);
    1116             :       // ARM mode needs an extra reg0 here due to addrmode2. Will go away once
    1117             :       // that refactoring is complete (eventually).
    1118         260 :       if (LdrOpc == ARM::LDR_POST_REG || LdrOpc == ARM::LDR_POST_IMM) {
    1119         189 :         MIB.addReg(0);
    1120         189 :         MIB.addImm(ARM_AM::getAM2Opc(ARM_AM::add, 4, ARM_AM::no_shift));
    1121             :       } else
    1122             :         MIB.addImm(4);
    1123         780 :       MIB.add(predOps(ARMCC::AL));
    1124             :     }
    1125        3559 :     Regs.clear();
    1126             : 
    1127             :     // Put any subsequent vpop instructions after this one: they will refer to
    1128             :     // higher register numbers so need to be popped afterwards.
    1129        7118 :     if (MI != MBB.end())
    1130             :       ++MI;
    1131             :   }
    1132        9717 : }
    1133             : 
    1134             : /// Emit aligned spill instructions for NumAlignedDPRCS2Regs D-registers
    1135             : /// starting from d8.  Also insert stack realignment code and leave the stack
    1136             : /// pointer pointing to the d8 spill slot.
    1137          39 : static void emitAlignedDPRCS2Spills(MachineBasicBlock &MBB,
    1138             :                                     MachineBasicBlock::iterator MI,
    1139             :                                     unsigned NumAlignedDPRCS2Regs,
    1140             :                                     const std::vector<CalleeSavedInfo> &CSI,
    1141             :                                     const TargetRegisterInfo *TRI) {
    1142          39 :   MachineFunction &MF = *MBB.getParent();
    1143          39 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1144         156 :   DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
    1145          39 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1146          39 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1147             : 
    1148             :   // Mark the D-register spill slots as properly aligned.  Since MFI computes
    1149             :   // stack slot layout backwards, this can actually mean that the d-reg stack
    1150             :   // slot offsets can be wrong. The offset for d8 will always be correct.
    1151         587 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
    1152        1018 :     unsigned DNum = CSI[i].getReg() - ARM::D8;
    1153         509 :     if (DNum > NumAlignedDPRCS2Regs - 1)
    1154         242 :       continue;
    1155         267 :     int FI = CSI[i].getFrameIdx();
    1156             :     // The even-numbered registers will be 16-byte aligned, the odd-numbered
    1157             :     // registers will be 8-byte aligned.
    1158         534 :     MFI.setObjectAlignment(FI, DNum % 2 ? 8 : 16);
    1159             : 
    1160             :     // The stack slot for D8 needs to be maximally aligned because this is
    1161             :     // actually the point where we align the stack pointer.  MachineFrameInfo
    1162             :     // computes all offsets relative to the incoming stack pointer which is a
    1163             :     // bit weird when realigning the stack.  Any extra padding for this
    1164             :     // over-alignment is not realized because the code inserted below adjusts
    1165             :     // the stack pointer by numregs * 8 before aligning the stack pointer.
    1166         267 :     if (DNum == 0)
    1167          39 :       MFI.setObjectAlignment(FI, MFI.getMaxAlignment());
    1168             :   }
    1169             : 
    1170             :   // Move the stack pointer to the d8 spill slot, and align it at the same
    1171             :   // time. Leave the stack slot address in the scratch register r4.
    1172             :   //
    1173             :   //   sub r4, sp, #numregs * 8
    1174             :   //   bic r4, r4, #align - 1
    1175             :   //   mov sp, r4
    1176             :   //
    1177          39 :   bool isThumb = AFI->isThumbFunction();
    1178             :   assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
    1179          39 :   AFI->setShouldRestoreSPFromFP(true);
    1180             : 
    1181             :   // sub r4, sp, #numregs * 8
    1182             :   // The immediate is <= 64, so it doesn't need any special encoding.
    1183          39 :   unsigned Opc = isThumb ? ARM::t2SUBri : ARM::SUBri;
    1184         117 :   BuildMI(MBB, MI, DL, TII.get(Opc), ARM::R4)
    1185          39 :       .addReg(ARM::SP)
    1186          78 :       .addImm(8 * NumAlignedDPRCS2Regs)
    1187         156 :       .add(predOps(ARMCC::AL))
    1188          78 :       .add(condCodeOp());
    1189             : 
    1190          39 :   unsigned MaxAlign = MF.getFrameInfo().getMaxAlignment();
    1191             :   // We must set parameter MustBeSingleInstruction to true, since
    1192             :   // skipAlignedDPRCS2Spills expects exactly 3 instructions to perform
    1193             :   // stack alignment.  Luckily, this can always be done since all ARM
    1194             :   // architecture versions that support Neon also support the BFC
    1195             :   // instruction.
    1196          39 :   emitAligningInstructions(MF, AFI, TII, MBB, MI, DL, ARM::R4, MaxAlign, true);
    1197             : 
    1198             :   // mov sp, r4
    1199             :   // The stack pointer must be adjusted before spilling anything, otherwise
    1200             :   // the stack slots could be clobbered by an interrupt handler.
    1201             :   // Leave r4 live, it is used below.
    1202          39 :   Opc = isThumb ? ARM::tMOVr : ARM::MOVr;
    1203         117 :   MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(Opc), ARM::SP)
    1204          39 :                                 .addReg(ARM::R4)
    1205         117 :                                 .add(predOps(ARMCC::AL));
    1206          39 :   if (!isThumb)
    1207          32 :     MIB.add(condCodeOp());
    1208             : 
    1209             :   // Now spill NumAlignedDPRCS2Regs registers starting from d8.
    1210             :   // r4 holds the stack slot address.
    1211          39 :   unsigned NextReg = ARM::D8;
    1212             : 
    1213             :   // 16-byte aligned vst1.64 with 4 d-regs and address writeback.
    1214             :   // The writeback is only needed when emitting two vst1.64 instructions.
    1215          39 :   if (NumAlignedDPRCS2Regs >= 6) {
    1216             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1217          62 :                                                &ARM::QQPRRegClass);
    1218          62 :     MBB.addLiveIn(SupReg);
    1219          93 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1d64Qwb_fixed), ARM::R4)
    1220          31 :         .addReg(ARM::R4, RegState::Kill)
    1221          31 :         .addImm(16)
    1222          31 :         .addReg(NextReg)
    1223          31 :         .addReg(SupReg, RegState::ImplicitKill)
    1224          93 :         .add(predOps(ARMCC::AL));
    1225          31 :     NextReg += 4;
    1226          31 :     NumAlignedDPRCS2Regs -= 4;
    1227             :   }
    1228             : 
    1229             :   // We won't modify r4 beyond this point.  It currently points to the next
    1230             :   // register to be spilled.
    1231          39 :   unsigned R4BaseReg = NextReg;
    1232             : 
    1233             :   // 16-byte aligned vst1.64 with 4 d-regs, no writeback.
    1234          39 :   if (NumAlignedDPRCS2Regs >= 4) {
    1235             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1236          62 :                                                &ARM::QQPRRegClass);
    1237          62 :     MBB.addLiveIn(SupReg);
    1238          93 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1d64Q))
    1239          31 :         .addReg(ARM::R4)
    1240          31 :         .addImm(16)
    1241          31 :         .addReg(NextReg)
    1242          31 :         .addReg(SupReg, RegState::ImplicitKill)
    1243          93 :         .add(predOps(ARMCC::AL));
    1244          31 :     NextReg += 4;
    1245          31 :     NumAlignedDPRCS2Regs -= 4;
    1246             :   }
    1247             : 
    1248             :   // 16-byte aligned vst1.64 with 2 d-regs.
    1249          39 :   if (NumAlignedDPRCS2Regs >= 2) {
    1250             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1251          16 :                                                &ARM::QPRRegClass);
    1252          16 :     MBB.addLiveIn(SupReg);
    1253          24 :     BuildMI(MBB, MI, DL, TII.get(ARM::VST1q64))
    1254           8 :         .addReg(ARM::R4)
    1255           8 :         .addImm(16)
    1256           8 :         .addReg(SupReg)
    1257          24 :         .add(predOps(ARMCC::AL));
    1258           8 :     NextReg += 2;
    1259           8 :     NumAlignedDPRCS2Regs -= 2;
    1260             :   }
    1261             : 
    1262             :   // Finally, use a vanilla vstr.64 for the odd last register.
    1263          39 :   if (NumAlignedDPRCS2Regs) {
    1264           6 :     MBB.addLiveIn(NextReg);
    1265             :     // vstr.64 uses addrmode5 which has an offset scale of 4.
    1266           9 :     BuildMI(MBB, MI, DL, TII.get(ARM::VSTRD))
    1267           3 :         .addReg(NextReg)
    1268           3 :         .addReg(ARM::R4)
    1269           6 :         .addImm((NextReg - R4BaseReg) * 2)
    1270           9 :         .add(predOps(ARMCC::AL));
    1271             :   }
    1272             : 
    1273             :   // The last spill instruction inserted should kill the scratch register r4.
    1274          78 :   std::prev(MI)->addRegisterKilled(ARM::R4, TRI);
    1275          39 : }
    1276             : 
    1277             : /// Skip past the code inserted by emitAlignedDPRCS2Spills, and return an
    1278             : /// iterator to the following instruction.
    1279             : static MachineBasicBlock::iterator
    1280          39 : skipAlignedDPRCS2Spills(MachineBasicBlock::iterator MI,
    1281             :                         unsigned NumAlignedDPRCS2Regs) {
    1282             :   //   sub r4, sp, #numregs * 8
    1283             :   //   bic r4, r4, #align - 1
    1284             :   //   mov sp, r4
    1285         117 :   ++MI; ++MI; ++MI;
    1286             :   assert(MI->mayStore() && "Expecting spill instruction");
    1287             : 
    1288             :   // These switches all fall through.
    1289          39 :   switch(NumAlignedDPRCS2Regs) {
    1290           1 :   case 7:
    1291             :     ++MI;
    1292             :     assert(MI->mayStore() && "Expecting spill instruction");
    1293             :     LLVM_FALLTHROUGH;
    1294          33 :   default:
    1295             :     ++MI;
    1296             :     assert(MI->mayStore() && "Expecting spill instruction");
    1297             :     LLVM_FALLTHROUGH;
    1298          39 :   case 1:
    1299             :   case 2:
    1300             :   case 4:
    1301             :     assert(MI->killsRegister(ARM::R4) && "Missed kill flag");
    1302          39 :     ++MI;
    1303             :   }
    1304          39 :   return MI;
    1305             : }
    1306             : 
    1307             : /// Emit aligned reload instructions for NumAlignedDPRCS2Regs D-registers
    1308             : /// starting from d8.  These instructions are assumed to execute while the
    1309             : /// stack is still aligned, unlike the code inserted by emitPopInst.
    1310          31 : static void emitAlignedDPRCS2Restores(MachineBasicBlock &MBB,
    1311             :                                       MachineBasicBlock::iterator MI,
    1312             :                                       unsigned NumAlignedDPRCS2Regs,
    1313             :                                       const std::vector<CalleeSavedInfo> &CSI,
    1314             :                                       const TargetRegisterInfo *TRI) {
    1315          31 :   MachineFunction &MF = *MBB.getParent();
    1316          31 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1317         124 :   DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
    1318          31 :   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
    1319             : 
    1320             :   // Find the frame index assigned to d8.
    1321          31 :   int D8SpillFI = 0;
    1322         412 :   for (unsigned i = 0, e = CSI.size(); i != e; ++i)
    1323         762 :     if (CSI[i].getReg() == ARM::D8) {
    1324          31 :       D8SpillFI = CSI[i].getFrameIdx();
    1325          31 :       break;
    1326             :     }
    1327             : 
    1328             :   // Materialize the address of the d8 spill slot into the scratch register r4.
    1329             :   // This can be fairly complicated if the stack frame is large, so just use
    1330             :   // the normal frame index elimination mechanism to do it.  This code runs as
    1331             :   // the initial part of the epilog where the stack and base pointers haven't
    1332             :   // been changed yet.
    1333          31 :   bool isThumb = AFI->isThumbFunction();
    1334             :   assert(!AFI->isThumb1OnlyFunction() && "Can't realign stack for thumb1");
    1335             : 
    1336          31 :   unsigned Opc = isThumb ? ARM::t2ADDri : ARM::ADDri;
    1337          62 :   BuildMI(MBB, MI, DL, TII.get(Opc), ARM::R4)
    1338          31 :       .addFrameIndex(D8SpillFI)
    1339          31 :       .addImm(0)
    1340         124 :       .add(predOps(ARMCC::AL))
    1341          62 :       .add(condCodeOp());
    1342             : 
    1343             :   // Now restore NumAlignedDPRCS2Regs registers starting from d8.
    1344          31 :   unsigned NextReg = ARM::D8;
    1345             : 
    1346             :   // 16-byte aligned vld1.64 with 4 d-regs and writeback.
    1347          31 :   if (NumAlignedDPRCS2Regs >= 6) {
    1348             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1349          46 :                                                &ARM::QQPRRegClass);
    1350          69 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1d64Qwb_fixed), NextReg)
    1351          23 :         .addReg(ARM::R4, RegState::Define)
    1352          23 :         .addReg(ARM::R4, RegState::Kill)
    1353          23 :         .addImm(16)
    1354          23 :         .addReg(SupReg, RegState::ImplicitDefine)
    1355          69 :         .add(predOps(ARMCC::AL));
    1356          23 :     NextReg += 4;
    1357          23 :     NumAlignedDPRCS2Regs -= 4;
    1358             :   }
    1359             : 
    1360             :   // We won't modify r4 beyond this point.  It currently points to the next
    1361             :   // register to be spilled.
    1362          31 :   unsigned R4BaseReg = NextReg;
    1363             : 
    1364             :   // 16-byte aligned vld1.64 with 4 d-regs, no writeback.
    1365          31 :   if (NumAlignedDPRCS2Regs >= 4) {
    1366             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1367          46 :                                                &ARM::QQPRRegClass);
    1368          69 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1d64Q), NextReg)
    1369          23 :         .addReg(ARM::R4)
    1370          23 :         .addImm(16)
    1371          23 :         .addReg(SupReg, RegState::ImplicitDefine)
    1372          69 :         .add(predOps(ARMCC::AL));
    1373          23 :     NextReg += 4;
    1374          23 :     NumAlignedDPRCS2Regs -= 4;
    1375             :   }
    1376             : 
    1377             :   // 16-byte aligned vld1.64 with 2 d-regs.
    1378          31 :   if (NumAlignedDPRCS2Regs >= 2) {
    1379             :     unsigned SupReg = TRI->getMatchingSuperReg(NextReg, ARM::dsub_0,
    1380          16 :                                                &ARM::QPRRegClass);
    1381          24 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLD1q64), SupReg)
    1382           8 :         .addReg(ARM::R4)
    1383           8 :         .addImm(16)
    1384          24 :         .add(predOps(ARMCC::AL));
    1385           8 :     NextReg += 2;
    1386           8 :     NumAlignedDPRCS2Regs -= 2;
    1387             :   }
    1388             : 
    1389             :   // Finally, use a vanilla vldr.64 for the remaining odd register.
    1390          31 :   if (NumAlignedDPRCS2Regs)
    1391           9 :     BuildMI(MBB, MI, DL, TII.get(ARM::VLDRD), NextReg)
    1392           3 :         .addReg(ARM::R4)
    1393           6 :         .addImm(2 * (NextReg - R4BaseReg))
    1394           9 :         .add(predOps(ARMCC::AL));
    1395             : 
    1396             :   // Last store kills r4.
    1397          62 :   std::prev(MI)->addRegisterKilled(ARM::R4, TRI);
    1398          31 : }
    1399             : 
    1400        3294 : bool ARMFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
    1401             :                                         MachineBasicBlock::iterator MI,
    1402             :                                         const std::vector<CalleeSavedInfo> &CSI,
    1403             :                                         const TargetRegisterInfo *TRI) const {
    1404        3294 :   if (CSI.empty())
    1405             :     return false;
    1406             : 
    1407        3294 :   MachineFunction &MF = *MBB.getParent();
    1408        3294 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1409             : 
    1410        3294 :   unsigned PushOpc = AFI->isThumbFunction() ? ARM::t2STMDB_UPD : ARM::STMDB_UPD;
    1411        3294 :   unsigned PushOneOpc = AFI->isThumbFunction() ?
    1412        3294 :     ARM::t2STR_PRE : ARM::STR_PRE_IMM;
    1413        3294 :   unsigned FltOpc = ARM::VSTMDDB_UPD;
    1414        3294 :   unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
    1415        3294 :   emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isARMArea1Register, 0,
    1416             :                MachineInstr::FrameSetup);
    1417        3294 :   emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isARMArea2Register, 0,
    1418             :                MachineInstr::FrameSetup);
    1419        3294 :   emitPushInst(MBB, MI, CSI, FltOpc, 0, true, &isARMArea3Register,
    1420             :                NumAlignedDPRCS2Regs, MachineInstr::FrameSetup);
    1421             : 
    1422             :   // The code above does not insert spill code for the aligned DPRCS2 registers.
    1423             :   // The stack realignment code will be inserted between the push instructions
    1424             :   // and these spills.
    1425        3294 :   if (NumAlignedDPRCS2Regs)
    1426          39 :     emitAlignedDPRCS2Spills(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
    1427             : 
    1428             :   return true;
    1429             : }
    1430             : 
    1431        3239 : bool ARMFrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
    1432             :                                         MachineBasicBlock::iterator MI,
    1433             :                                         std::vector<CalleeSavedInfo> &CSI,
    1434             :                                         const TargetRegisterInfo *TRI) const {
    1435        3239 :   if (CSI.empty())
    1436             :     return false;
    1437             : 
    1438        3239 :   MachineFunction &MF = *MBB.getParent();
    1439        3239 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1440        3239 :   bool isVarArg = AFI->getArgRegsSaveSize() > 0;
    1441        3239 :   unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs();
    1442             : 
    1443             :   // The emitPopInst calls below do not insert reloads for the aligned DPRCS2
    1444             :   // registers. Do that here instead.
    1445        3239 :   if (NumAlignedDPRCS2Regs)
    1446          31 :     emitAlignedDPRCS2Restores(MBB, MI, NumAlignedDPRCS2Regs, CSI, TRI);
    1447             : 
    1448        3239 :   unsigned PopOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_UPD : ARM::LDMIA_UPD;
    1449        3239 :   unsigned LdrOpc = AFI->isThumbFunction() ? ARM::t2LDR_POST :ARM::LDR_POST_IMM;
    1450        3239 :   unsigned FltOpc = ARM::VLDMDIA_UPD;
    1451        3239 :   emitPopInst(MBB, MI, CSI, FltOpc, 0, isVarArg, true, &isARMArea3Register,
    1452             :               NumAlignedDPRCS2Regs);
    1453        3239 :   emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false,
    1454             :               &isARMArea2Register, 0);
    1455        3239 :   emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false,
    1456             :               &isARMArea1Register, 0);
    1457             : 
    1458        3239 :   return true;
    1459             : }
    1460             : 
    1461             : // FIXME: Make generic?
    1462         484 : static unsigned GetFunctionSizeInBytes(const MachineFunction &MF,
    1463             :                                        const ARMBaseInstrInfo &TII) {
    1464         484 :   unsigned FnSize = 0;
    1465        2514 :   for (auto &MBB : MF) {
    1466       13736 :     for (auto &MI : MBB)
    1467        4744 :       FnSize += TII.getInstSizeInBytes(MI);
    1468             :   }
    1469         484 :   return FnSize;
    1470             : }
    1471             : 
    1472             : /// estimateRSStackSizeLimit - Look at each instruction that references stack
    1473             : /// frames and return the stack size limit beyond which some of these
    1474             : /// instructions will require a scratch register during their expansion later.
    1475             : // FIXME: Move to TII?
    1476       12002 : static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
    1477             :                                          const TargetFrameLowering *TFI) {
    1478       12002 :   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1479       12002 :   unsigned Limit = (1 << 12) - 1;
    1480       53290 :   for (auto &MBB : MF) {
    1481      277360 :     for (auto &MI : MBB) {
    1482      538679 :       for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    1483     1344594 :         if (!MI.getOperand(i).isFI())
    1484             :           continue;
    1485             : 
    1486             :         // When using ADDri to get the address of a stack object, 255 is the
    1487             :         // largest offset guaranteed to fit in the immediate offset.
    1488       13642 :         if (MI.getOpcode() == ARM::ADDri) {
    1489        2422 :           Limit = std::min(Limit, (1U << 8) - 1);
    1490        1211 :           break;
    1491             :         }
    1492             : 
    1493             :         // Otherwise check the addressing mode.
    1494       12431 :         switch (MI.getDesc().TSFlags & ARMII::AddrModeMask) {
    1495          61 :         case ARMII::AddrMode3:
    1496             :         case ARMII::AddrModeT2_i8:
    1497         122 :           Limit = std::min(Limit, (1U << 8) - 1);
    1498          61 :           break;
    1499        1095 :         case ARMII::AddrMode5:
    1500             :         case ARMII::AddrModeT2_i8s4:
    1501        2190 :           Limit = std::min(Limit, ((1U << 8) - 1) * 4);
    1502        1095 :           break;
    1503        2960 :         case ARMII::AddrModeT2_i12:
    1504             :           // i12 supports only positive offset so these will be converted to
    1505             :           // i8 opcodes. See llvm::rewriteT2FrameIndex.
    1506        2960 :           if (TFI->hasFP(MF) && AFI->hasStackFrame())
    1507           0 :             Limit = std::min(Limit, (1U << 8) - 1);
    1508             :           break;
    1509          22 :         case ARMII::AddrMode4:
    1510             :         case ARMII::AddrMode6:
    1511             :           // Addressing modes 4 & 6 (load/store) instructions can't encode an
    1512             :           // immediate offset for stack references.
    1513          22 :           return 0;
    1514             :         default:
    1515             :           break;
    1516             :         }
    1517             :         break; // At most one FI per instruction
    1518             :       }
    1519             :     }
    1520             :   }
    1521             : 
    1522       11980 :   return Limit;
    1523             : }
    1524             : 
    1525             : // In functions that realign the stack, it can be an advantage to spill the
    1526             : // callee-saved vector registers after realigning the stack. The vst1 and vld1
    1527             : // instructions take alignment hints that can improve performance.
    1528             : //
    1529             : static void
    1530       12002 : checkNumAlignedDPRCS2Regs(MachineFunction &MF, BitVector &SavedRegs) {
    1531       24004 :   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(0);
    1532       12002 :   if (!SpillAlignedNEONRegs)
    1533             :     return;
    1534             : 
    1535             :   // Naked functions don't spill callee-saved registers.
    1536       23998 :   if (MF.getFunction()->hasFnAttribute(Attribute::Naked))
    1537             :     return;
    1538             : 
    1539             :   // We are planning to use NEON instructions vst1 / vld1.
    1540       11998 :   if (!static_cast<const ARMSubtarget &>(MF.getSubtarget()).hasNEON())
    1541             :     return;
    1542             : 
    1543             :   // Don't bother if the default stack alignment is sufficiently high.
    1544        7040 :   if (MF.getSubtarget().getFrameLowering()->getStackAlignment() >= 8)
    1545             :     return;
    1546             : 
    1547             :   // Aligned spills require stack realignment.
    1548        1767 :   if (!static_cast<const ARMBaseRegisterInfo *>(
    1549        1767 :            MF.getSubtarget().getRegisterInfo())->canRealignStack(MF))
    1550             :     return;
    1551             : 
    1552             :   // We always spill contiguous d-registers starting from d8. Count how many
    1553             :   // needs spilling.  The register allocator will almost always use the
    1554             :   // callee-saved registers in order, but it can happen that there are holes in
    1555             :   // the range.  Registers above the hole will be spilled to the standard DPRCS
    1556             :   // area.
    1557             :   unsigned NumSpills = 0;
    1558        1362 :   for (; NumSpills < 8; ++NumSpills)
    1559        2120 :     if (!SavedRegs.test(ARM::D8 + NumSpills))
    1560             :       break;
    1561             : 
    1562             :   // Don't do this for just one d-register. It's not worth it.
    1563         818 :   if (NumSpills < 2)
    1564             :     return;
    1565             : 
    1566             :   // Spill the first NumSpills D-registers after realigning the stack.
    1567          78 :   MF.getInfo<ARMFunctionInfo>()->setNumAlignedDPRCS2Regs(NumSpills);
    1568             : 
    1569             :   // A scratch register is required for the vst1 / vld1 instructions.
    1570             :   SavedRegs.set(ARM::R4);
    1571             : }
    1572             : 
    1573       12002 : void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
    1574             :                                             BitVector &SavedRegs,
    1575             :                                             RegScavenger *RS) const {
    1576       12002 :   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
    1577             :   // This tells PEI to spill the FP as if it is any other callee-save register
    1578             :   // to take advantage the eliminateFrameIndex machinery. This also ensures it
    1579             :   // is spilled in the order specified by getCalleeSavedRegs() to make it easier
    1580             :   // to combine multiple loads / stores.
    1581       12002 :   bool CanEliminateFrame = true;
    1582       12002 :   bool CS1Spilled = false;
    1583       12002 :   bool LRSpilled = false;
    1584       12002 :   unsigned NumGPRSpills = 0;
    1585       12002 :   unsigned NumFPRSpills = 0;
    1586       24004 :   SmallVector<unsigned, 4> UnspilledCS1GPRs;
    1587       24004 :   SmallVector<unsigned, 4> UnspilledCS2GPRs;
    1588             :   const ARMBaseRegisterInfo *RegInfo = static_cast<const ARMBaseRegisterInfo *>(
    1589       12002 :       MF.getSubtarget().getRegisterInfo());
    1590             :   const ARMBaseInstrInfo &TII =
    1591       12002 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    1592       12002 :   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    1593       12002 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    1594       12002 :   MachineRegisterInfo &MRI = MF.getRegInfo();
    1595       12002 :   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
    1596             :   (void)TRI;  // Silence unused warning in non-assert builds.
    1597       12002 :   unsigned FramePtr = RegInfo->getFrameRegister(MF);
    1598             : 
    1599             :   // Spill R4 if Thumb2 function requires stack realignment - it will be used as
    1600             :   // scratch register. Also spill R4 if Thumb2 function has varsized objects,
    1601             :   // since it's not always possible to restore sp from fp in a single
    1602             :   // instruction.
    1603             :   // FIXME: It will be better just to find spare register here.
    1604       15834 :   if (AFI->isThumb2Function() &&
    1605        3818 :       (MFI.hasVarSizedObjects() || RegInfo->needsStackRealignment(MF)))
    1606             :     SavedRegs.set(ARM::R4);
    1607             : 
    1608       12002 :   if (AFI->isThumb1OnlyFunction()) {
    1609             :     // Spill LR if Thumb1 function uses variable length argument lists.
    1610         912 :     if (AFI->getArgRegsSaveSize() > 0)
    1611             :       SavedRegs.set(ARM::LR);
    1612             : 
    1613             :     // Spill R4 if Thumb1 epilogue has to restore SP from FP. We don't know
    1614             :     // for sure what the stack size will be, but for this, an estimate is good
    1615             :     // enough. If there anything changes it, it'll be a spill, which implies
    1616             :     // we've used all the registers and so R4 is already used, so not marking
    1617             :     // it here will be OK.
    1618             :     // FIXME: It will be better just to find spare register here.
    1619         912 :     unsigned StackSize = MFI.estimateStackSize(MF);
    1620         912 :     if (MFI.hasVarSizedObjects() || StackSize > 508)
    1621             :       SavedRegs.set(ARM::R4);
    1622             :   }
    1623             : 
    1624             :   // See if we can spill vector registers to aligned stack.
    1625       12002 :   checkNumAlignedDPRCS2Regs(MF, SavedRegs);
    1626             : 
    1627             :   // Spill the BasePtr if it's used.
    1628       12002 :   if (RegInfo->hasBasePointer(MF))
    1629          20 :     SavedRegs.set(RegInfo->getBaseRegister());
    1630             : 
    1631             :   // Don't spill FP if the frame can be eliminated. This is determined
    1632             :   // by scanning the callee-save registers to see if any is modified.
    1633       12002 :   const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF);
    1634      213964 :   for (unsigned i = 0; CSRegs[i]; ++i) {
    1635      201962 :     unsigned Reg = CSRegs[i];
    1636      201962 :     bool Spilled = false;
    1637      403924 :     if (SavedRegs.test(Reg)) {
    1638        8815 :       Spilled = true;
    1639        8815 :       CanEliminateFrame = false;
    1640             :     }
    1641             : 
    1642      500188 :     if (!ARM::GPRRegClass.contains(Reg)) {
    1643       96264 :       if (Spilled) {
    1644        2344 :         if (ARM::SPRRegClass.contains(Reg))
    1645           0 :           NumFPRSpills++;
    1646        2344 :         else if (ARM::DPRRegClass.contains(Reg))
    1647        1172 :           NumFPRSpills += 2;
    1648           0 :         else if (ARM::QPRRegClass.contains(Reg))
    1649           0 :           NumFPRSpills += 4;
    1650             :       }
    1651      281617 :       continue;
    1652             :     }
    1653             : 
    1654      105698 :     if (Spilled) {
    1655        7643 :       NumGPRSpills++;
    1656             : 
    1657       13191 :       if (!STI.splitFramePushPop(MF)) {
    1658        5548 :         if (Reg == ARM::LR)
    1659        2619 :           LRSpilled = true;
    1660        5548 :         CS1Spilled = true;
    1661        5548 :         continue;
    1662             :       }
    1663             : 
    1664             :       // Keep track if LR and any of R4, R5, R6, and R7 is spilled.
    1665        2095 :       switch (Reg) {
    1666         901 :       case ARM::LR:
    1667         901 :         LRSpilled = true;
    1668             :         LLVM_FALLTHROUGH;
    1669             :       case ARM::R0: case ARM::R1:
    1670             :       case ARM::R2: case ARM::R3:
    1671             :       case ARM::R4: case ARM::R5:
    1672             :       case ARM::R6: case ARM::R7:
    1673             :         CS1Spilled = true;
    1674             :         break;
    1675             :       default:
    1676             :         break;
    1677             :       }
    1678             :     } else {
    1679      181596 :       if (!STI.splitFramePushPop(MF)) {
    1680       83541 :         UnspilledCS1GPRs.push_back(Reg);
    1681       83541 :         continue;
    1682             :       }
    1683             : 
    1684       14514 :       switch (Reg) {
    1685        7989 :       case ARM::R0: case ARM::R1:
    1686             :       case ARM::R2: case ARM::R3:
    1687             :       case ARM::R4: case ARM::R5:
    1688             :       case ARM::R6: case ARM::R7:
    1689             :       case ARM::LR:
    1690        7989 :         UnspilledCS1GPRs.push_back(Reg);
    1691        7989 :         break;
    1692        6525 :       default:
    1693        6525 :         UnspilledCS2GPRs.push_back(Reg);
    1694        6525 :         break;
    1695             :       }
    1696             :     }
    1697             :   }
    1698             : 
    1699       12002 :   bool ForceLRSpill = false;
    1700       12002 :   if (!LRSpilled && AFI->isThumb1OnlyFunction()) {
    1701         484 :     unsigned FnSize = GetFunctionSizeInBytes(MF, TII);
    1702             :     // Force LR to be spilled if the Thumb function size is > 2048. This enables
    1703             :     // use of BL to implement far jump. If it turns out that it's not needed
    1704             :     // then the branch fix up path will undo it.
    1705         484 :     if (FnSize >= (1 << 11)) {
    1706           1 :       CanEliminateFrame = false;
    1707           1 :       ForceLRSpill = true;
    1708             :     }
    1709             :   }
    1710             : 
    1711             :   // If any of the stack slot references may be out of range of an immediate
    1712             :   // offset, make sure a register (or a spill slot) is available for the
    1713             :   // register scavenger. Note that if we're indexing off the frame pointer, the
    1714             :   // effective stack size is 4 bytes larger since the FP points to the stack
    1715             :   // slot of the previous FP. Also, if we have variable sized objects in the
    1716             :   // function, stack slot references will often be negative, and some of
    1717             :   // our instructions are positive-offset only, so conservatively consider
    1718             :   // that case to want a spill slot (or register) as well. Similarly, if
    1719             :   // the function adjusts the stack pointer during execution and the
    1720             :   // adjustments aren't already part of our stack size estimate, our offset
    1721             :   // calculations may be off, so be conservative.
    1722             :   // FIXME: We could add logic to be more precise about negative offsets
    1723             :   //        and which instructions will need a scratch register for them. Is it
    1724             :   //        worth the effort and added fragility?
    1725             :   unsigned EstimatedStackSize =
    1726       12002 :       MFI.estimateStackSize(MF) + 4 * (NumGPRSpills + NumFPRSpills);
    1727             : 
    1728             :   // Determine biggest (positive) SP offset in MachineFrameInfo.
    1729       12002 :   int MaxFixedOffset = 0;
    1730       13458 :   for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
    1731        2912 :     int MaxObjectOffset = MFI.getObjectOffset(I) + MFI.getObjectSize(I);
    1732        1456 :     MaxFixedOffset = std::max(MaxFixedOffset, MaxObjectOffset);
    1733             :   }
    1734             : 
    1735       12002 :   bool HasFP = hasFP(MF);
    1736       12002 :   if (HasFP) {
    1737        1852 :     if (AFI->hasStackFrame())
    1738           3 :       EstimatedStackSize += 4;
    1739             :   } else {
    1740             :     // If FP is not used, SP will be used to access arguments, so count the
    1741             :     // size of arguments into the estimation.
    1742       10150 :     EstimatedStackSize += MaxFixedOffset;
    1743             :   }
    1744       12002 :   EstimatedStackSize += 16; // For possible paddings.
    1745             : 
    1746       12002 :   unsigned EstimatedRSStackSizeLimit = estimateRSStackSizeLimit(MF, this);
    1747       24004 :   int MaxFPOffset = getMaxFPOffset(*MF.getFunction(), *AFI);
    1748       11846 :   bool BigFrameOffsets = EstimatedStackSize >= EstimatedRSStackSizeLimit ||
    1749       11806 :     MFI.hasVarSizedObjects() ||
    1750       15186 :     (MFI.adjustsStack() && !canSimplifyCallFramePseudos(MF)) ||
    1751             :     // For large argument stacks fp relative addressed may overflow.
    1752        1752 :     (HasFP && (MaxFixedOffset - MaxFPOffset) >= (int)EstimatedRSStackSizeLimit);
    1753       12002 :   if (BigFrameOffsets ||
    1754       24004 :       !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF)) {
    1755        3858 :     AFI->setHasStackFrame(true);
    1756             : 
    1757        3858 :     if (HasFP) {
    1758        2144 :       SavedRegs.set(FramePtr);
    1759             :       // If the frame pointer is required by the ABI, also spill LR so that we
    1760             :       // emit a complete frame record.
    1761        1072 :       if (MF.getTarget().Options.DisableFramePointerElim(MF) && !LRSpilled) {
    1762         116 :         SavedRegs.set(ARM::LR);
    1763         116 :         LRSpilled = true;
    1764         116 :         NumGPRSpills++;
    1765         232 :         auto LRPos = llvm::find(UnspilledCS1GPRs, ARM::LR);
    1766         116 :         if (LRPos != UnspilledCS1GPRs.end())
    1767         116 :           UnspilledCS1GPRs.erase(LRPos);
    1768             :       }
    1769        1072 :       auto FPPos = llvm::find(UnspilledCS1GPRs, FramePtr);
    1770        1072 :       if (FPPos != UnspilledCS1GPRs.end())
    1771        1018 :         UnspilledCS1GPRs.erase(FPPos);
    1772        1072 :       NumGPRSpills++;
    1773        1072 :       if (FramePtr == ARM::R7)
    1774         764 :         CS1Spilled = true;
    1775             :     }
    1776             : 
    1777             :     // This is true when we inserted a spill for an unused register that can now
    1778             :     // be used for register scavenging.
    1779        3858 :     bool ExtraCSSpill = false;
    1780             : 
    1781        3858 :     if (AFI->isThumb1OnlyFunction()) {
    1782             :       // For Thumb1-only targets, we need some low registers when we save and
    1783             :       // restore the high registers (which aren't allocatable, but could be
    1784             :       // used by inline assembly) because the push/pop instructions can not
    1785             :       // access high registers. If necessary, we might need to push more low
    1786             :       // registers to ensure that there is at least one free that can be used
    1787             :       // for the saving & restoring, and preferably we should ensure that as
    1788             :       // many as are needed are available so that fewer push/pop instructions
    1789             :       // are required.
    1790             : 
    1791             :       // Low registers which are not currently pushed, but could be (r4-r7).
    1792        1078 :       SmallVector<unsigned, 4> AvailableRegs;
    1793             : 
    1794             :       // Unused argument registers (r0-r3) can be clobbered in the prologue for
    1795             :       // free.
    1796         539 :       int EntryRegDeficit = 0;
    1797        3234 :       for (unsigned Reg : {ARM::R0, ARM::R1, ARM::R2, ARM::R3}) {
    1798        2156 :         if (!MF.getRegInfo().isLiveIn(Reg)) {
    1799        1633 :           --EntryRegDeficit;
    1800             :           DEBUG(dbgs() << PrintReg(Reg, TRI)
    1801             :                        << " is unused argument register, EntryRegDeficit = "
    1802             :                        << EntryRegDeficit << "\n");
    1803             :         }
    1804             :       }
    1805             : 
    1806             :       // Unused return registers can be clobbered in the epilogue for free.
    1807         539 :       int ExitRegDeficit = AFI->getReturnRegsCount() - 4;
    1808             :       DEBUG(dbgs() << AFI->getReturnRegsCount()
    1809             :                    << " return regs used, ExitRegDeficit = " << ExitRegDeficit
    1810             :                    << "\n");
    1811             : 
    1812         539 :       int RegDeficit = std::max(EntryRegDeficit, ExitRegDeficit);
    1813             :       DEBUG(dbgs() << "RegDeficit = " << RegDeficit << "\n");
    1814             : 
    1815             :       // r4-r6 can be used in the prologue if they are pushed by the first push
    1816             :       // instruction.
    1817        2695 :       for (unsigned Reg : {ARM::R4, ARM::R5, ARM::R6}) {
    1818        3234 :         if (SavedRegs.test(Reg)) {
    1819         486 :           --RegDeficit;
    1820             :           DEBUG(dbgs() << PrintReg(Reg, TRI)
    1821             :                        << " is saved low register, RegDeficit = " << RegDeficit
    1822             :                        << "\n");
    1823             :         } else {
    1824        1131 :           AvailableRegs.push_back(Reg);
    1825             :           DEBUG(dbgs()
    1826             :                 << PrintReg(Reg, TRI)
    1827             :                 << " is non-saved low register, adding to AvailableRegs\n");
    1828             :         }
    1829             :       }
    1830             : 
    1831             :       // r7 can be used if it is not being used as the frame pointer.
    1832         539 :       if (!HasFP) {
    1833         422 :         if (SavedRegs.test(ARM::R7)) {
    1834          28 :           --RegDeficit;
    1835             :           DEBUG(dbgs() << "%R7 is saved low register, RegDeficit = "
    1836             :                        << RegDeficit << "\n");
    1837             :         } else {
    1838         394 :           AvailableRegs.push_back(ARM::R7);
    1839             :           DEBUG(dbgs()
    1840             :                 << "%R7 is non-saved low register, adding to AvailableRegs\n");
    1841             :         }
    1842             :       }
    1843             : 
    1844             :       // Each of r8-r11 needs to be copied to a low register, then pushed.
    1845        4851 :       for (unsigned Reg : {ARM::R8, ARM::R9, ARM::R10, ARM::R11}) {
    1846        2156 :         if (SavedRegs.test(Reg)) {
    1847          54 :           ++RegDeficit;
    1848             :           DEBUG(dbgs() << PrintReg(Reg, TRI)
    1849             :                        << " is saved high register, RegDeficit = " << RegDeficit
    1850             :                        << "\n");
    1851             :         }
    1852             :       }
    1853             : 
    1854             :       // LR can only be used by PUSH, not POP, and can't be used at all if the
    1855             :       // llvm.returnaddress intrinsic is used. This is only worth doing if we
    1856             :       // are more limited at function entry than exit.
    1857         713 :       if ((EntryRegDeficit > ExitRegDeficit) &&
    1858         174 :           !(MF.getRegInfo().isLiveIn(ARM::LR) &&
    1859           0 :             MF.getFrameInfo().isReturnAddressTaken())) {
    1860         174 :         if (SavedRegs.test(ARM::LR)) {
    1861         132 :           --RegDeficit;
    1862             :           DEBUG(dbgs() << "%LR is saved register, RegDeficit = " << RegDeficit
    1863             :                        << "\n");
    1864             :         } else {
    1865          42 :           AvailableRegs.push_back(ARM::LR);
    1866             :           DEBUG(dbgs() << "%LR is not saved, adding to AvailableRegs\n");
    1867             :         }
    1868             :       }
    1869             : 
    1870             :       // If there are more high registers that need pushing than low registers
    1871             :       // available, push some more low registers so that we can use fewer push
    1872             :       // instructions. This might not reduce RegDeficit all the way to zero,
    1873             :       // because we can only guarantee that r4-r6 are available, but r8-r11 may
    1874             :       // need saving.
    1875             :       DEBUG(dbgs() << "Final RegDeficit = " << RegDeficit << "\n");
    1876         555 :       for (; RegDeficit > 0 && !AvailableRegs.empty(); --RegDeficit) {
    1877           8 :         unsigned Reg = AvailableRegs.pop_back_val();
    1878             :         DEBUG(dbgs() << "Spilling " << PrintReg(Reg, TRI)
    1879             :                      << " to make up reg deficit\n");
    1880          16 :         SavedRegs.set(Reg);
    1881           8 :         NumGPRSpills++;
    1882           8 :         CS1Spilled = true;
    1883             :         assert(!MRI.isReserved(Reg) && "Should not be reserved");
    1884           8 :         if (!MRI.isPhysRegUsed(Reg))
    1885           8 :           ExtraCSSpill = true;
    1886           8 :         UnspilledCS1GPRs.erase(llvm::find(UnspilledCS1GPRs, Reg));
    1887           8 :         if (Reg == ARM::LR)
    1888           1 :           LRSpilled = true;
    1889             :       }
    1890             :       DEBUG(dbgs() << "After adding spills, RegDeficit = " << RegDeficit << "\n");
    1891             :     }
    1892             : 
    1893             :     // If LR is not spilled, but at least one of R4, R5, R6, and R7 is spilled.
    1894             :     // Spill LR as well so we can fold BX_RET to the registers restore (LDM).
    1895        3858 :     if (!LRSpilled && CS1Spilled) {
    1896         181 :       SavedRegs.set(ARM::LR);
    1897         181 :       NumGPRSpills++;
    1898             :       SmallVectorImpl<unsigned>::iterator LRPos;
    1899         362 :       LRPos = llvm::find(UnspilledCS1GPRs, (unsigned)ARM::LR);
    1900         181 :       if (LRPos != UnspilledCS1GPRs.end())
    1901         181 :         UnspilledCS1GPRs.erase(LRPos);
    1902             : 
    1903         181 :       ForceLRSpill = false;
    1904         181 :       if (!MRI.isReserved(ARM::LR) && !MRI.isPhysRegUsed(ARM::LR))
    1905             :         ExtraCSSpill = true;
    1906             :     }
    1907             : 
    1908             :     // If stack and double are 8-byte aligned and we are spilling an odd number
    1909             :     // of GPRs, spill one extra callee save GPR so we won't have to pad between
    1910             :     // the integer and double callee save areas.
    1911             :     DEBUG(dbgs() << "NumGPRSpills = " << NumGPRSpills << "\n");
    1912        3858 :     unsigned TargetAlign = getStackAlignment();
    1913        3858 :     if (TargetAlign >= 8 && (NumGPRSpills & 1)) {
    1914        1827 :       if (CS1Spilled && !UnspilledCS1GPRs.empty()) {
    1915        4989 :         for (unsigned i = 0, e = UnspilledCS1GPRs.size(); i != e; ++i) {
    1916        6490 :           unsigned Reg = UnspilledCS1GPRs[i];
    1917             :           // Don't spill high register if the function is thumb.  In the case of
    1918             :           // Windows on ARM, accept R11 (frame pointer)
    1919        5424 :           if (!AFI->isThumbFunction() ||
    1920        4439 :               (STI.isTargetWindows() && Reg == ARM::R11) ||
    1921        4760 :               isARMLowRegister(Reg) || Reg == ARM::LR) {
    1922        1730 :             SavedRegs.set(Reg);
    1923             :             DEBUG(dbgs() << "Spilling " << PrintReg(Reg, TRI)
    1924             :                          << " to make up alignment\n");
    1925        1730 :             if (!MRI.isReserved(Reg) && !MRI.isPhysRegUsed(Reg))
    1926             :               ExtraCSSpill = true;
    1927             :             break;
    1928             :           }
    1929             :         }
    1930         121 :       } else if (!UnspilledCS2GPRs.empty() && !AFI->isThumb1OnlyFunction()) {
    1931          16 :         unsigned Reg = UnspilledCS2GPRs.front();
    1932           8 :         SavedRegs.set(Reg);
    1933             :         DEBUG(dbgs() << "Spilling " << PrintReg(Reg, TRI)
    1934             :                      << " to make up alignment\n");
    1935           8 :         if (!MRI.isReserved(Reg) && !MRI.isPhysRegUsed(Reg))
    1936             :           ExtraCSSpill = true;
    1937             :       }
    1938             :     }
    1939             : 
    1940             :     // Estimate if we might need to scavenge a register at some point in order
    1941             :     // to materialize a stack offset. If so, either spill one additional
    1942             :     // callee-saved register or reserve a special spill slot to facilitate
    1943             :     // register scavenging. Thumb1 needs a spill slot for stack pointer
    1944             :     // adjustments also, even when the frame itself is small.
    1945        3858 :     if (BigFrameOffsets && !ExtraCSSpill) {
    1946             :       // If any non-reserved CS register isn't spilled, just spill one or two
    1947             :       // extra. That should take care of it!
    1948         114 :       unsigned NumExtras = TargetAlign / 4;
    1949         114 :       SmallVector<unsigned, 2> Extras;
    1950         404 :       while (NumExtras && !UnspilledCS1GPRs.empty()) {
    1951         290 :         unsigned Reg = UnspilledCS1GPRs.back();
    1952         145 :         UnspilledCS1GPRs.pop_back();
    1953         290 :         if (!MRI.isReserved(Reg) &&
    1954         172 :             (!AFI->isThumb1OnlyFunction() || isARMLowRegister(Reg) ||
    1955             :              Reg == ARM::LR)) {
    1956         145 :           Extras.push_back(Reg);
    1957         145 :           NumExtras--;
    1958             :         }
    1959             :       }
    1960             :       // For non-Thumb1 functions, also check for hi-reg CS registers
    1961         114 :       if (!AFI->isThumb1OnlyFunction()) {
    1962          94 :         while (NumExtras && !UnspilledCS2GPRs.empty()) {
    1963           4 :           unsigned Reg = UnspilledCS2GPRs.back();
    1964           2 :           UnspilledCS2GPRs.pop_back();
    1965           4 :           if (!MRI.isReserved(Reg)) {
    1966           2 :             Extras.push_back(Reg);
    1967           2 :             NumExtras--;
    1968             :           }
    1969             :         }
    1970             :       }
    1971         114 :       if (NumExtras == 0) {
    1972         399 :         for (unsigned Reg : Extras) {
    1973         138 :           SavedRegs.set(Reg);
    1974         138 :           if (!MRI.isPhysRegUsed(Reg))
    1975         138 :             ExtraCSSpill = true;
    1976             :         }
    1977             :       }
    1978         114 :       if (!ExtraCSSpill && !AFI->isThumb1OnlyFunction()) {
    1979             :         // note: Thumb1 functions spill to R12, not the stack.  Reserve a slot
    1980             :         // closest to SP or frame pointer.
    1981             :         assert(RS && "Register scavenging not provided");
    1982          17 :         const TargetRegisterClass &RC = ARM::GPRRegClass;
    1983          17 :         unsigned Size = TRI->getSpillSize(RC);
    1984          34 :         unsigned Align = TRI->getSpillAlignment(RC);
    1985          17 :         RS->addScavengingFrameIndex(MFI.CreateStackObject(Size, Align, false));
    1986             :       }
    1987             :     }
    1988             :   }
    1989             : 
    1990       12002 :   if (ForceLRSpill) {
    1991           1 :     SavedRegs.set(ARM::LR);
    1992             :     AFI->setLRIsSpilledForFarJump(true);
    1993             :   }
    1994       12002 : }
    1995             : 
    1996       12038 : MachineBasicBlock::iterator ARMFrameLowering::eliminateCallFramePseudoInstr(
    1997             :     MachineFunction &MF, MachineBasicBlock &MBB,
    1998             :     MachineBasicBlock::iterator I) const {
    1999             :   const ARMBaseInstrInfo &TII =
    2000       12038 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    2001       12038 :   if (!hasReservedCallFrame(MF)) {
    2002             :     // If we have alloca, convert as follows:
    2003             :     // ADJCALLSTACKDOWN -> sub, sp, sp, amount
    2004             :     // ADJCALLSTACKUP   -> add, sp, sp, amount
    2005         158 :     MachineInstr &Old = *I;
    2006         474 :     DebugLoc dl = Old.getDebugLoc();
    2007         316 :     unsigned Amount = TII.getFrameSize(Old);
    2008         158 :     if (Amount != 0) {
    2009             :       // We need to keep the stack aligned properly.  To do this, we round the
    2010             :       // amount of space needed for the outgoing arguments up to the next
    2011             :       // alignment boundary.
    2012          84 :       Amount = alignSPAdjust(Amount);
    2013             : 
    2014          42 :       ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    2015             :       assert(!AFI->isThumb1OnlyFunction() &&
    2016             :              "This eliminateCallFramePseudoInstr does not support Thumb1!");
    2017          42 :       bool isARM = !AFI->isThumbFunction();
    2018             : 
    2019             :       // Replace the pseudo instruction with a new instruction...
    2020          42 :       unsigned Opc = Old.getOpcode();
    2021          42 :       int PIdx = Old.findFirstPredOperandIdx();
    2022             :       ARMCC::CondCodes Pred =
    2023          84 :           (PIdx == -1) ? ARMCC::AL
    2024         126 :                        : (ARMCC::CondCodes)Old.getOperand(PIdx).getImm();
    2025          84 :       unsigned PredReg = TII.getFramePred(Old);
    2026          42 :       if (Opc == ARM::ADJCALLSTACKDOWN || Opc == ARM::tADJCALLSTACKDOWN) {
    2027          21 :         emitSPUpdate(isARM, MBB, I, dl, TII, -Amount, MachineInstr::NoFlags,
    2028             :                      Pred, PredReg);
    2029             :       } else {
    2030             :         assert(Opc == ARM::ADJCALLSTACKUP || Opc == ARM::tADJCALLSTACKUP);
    2031          21 :         emitSPUpdate(isARM, MBB, I, dl, TII, Amount, MachineInstr::NoFlags,
    2032             :                      Pred, PredReg);
    2033             :       }
    2034             :     }
    2035             :   }
    2036       12038 :   return MBB.erase(I);
    2037             : }
    2038             : 
    2039             : /// Get the minimum constant for ARM that is greater than or equal to the
    2040             : /// argument. In ARM, constants can have any value that can be produced by
    2041             : /// rotating an 8-bit value to the right by an even number of bits within a
    2042             : /// 32-bit word.
    2043         104 : static uint32_t alignToARMConstant(uint32_t Value) {
    2044         104 :   unsigned Shifted = 0;
    2045             : 
    2046         104 :   if (Value == 0)
    2047             :       return 0;
    2048             : 
    2049        1244 :   while (!(Value & 0xC0000000)) {
    2050         596 :       Value = Value << 2;
    2051         596 :       Shifted += 2;
    2052             :   }
    2053             : 
    2054          52 :   bool Carry = (Value & 0x00FFFFFF);
    2055          52 :   Value = ((Value & 0xFF000000) >> 24) + Carry;
    2056             : 
    2057          52 :   if (Value & 0x0000100)
    2058           0 :       Value = Value & 0x000001FC;
    2059             : 
    2060          52 :   if (Shifted > 24)
    2061          36 :       Value = Value >> (Shifted - 24);
    2062             :   else
    2063          16 :       Value = Value << (24 - Shifted);
    2064             : 
    2065             :   return Value;
    2066             : }
    2067             : 
    2068             : // The stack limit in the TCB is set to this many bytes above the actual
    2069             : // stack limit.
    2070             : static const uint64_t kSplitStackAvailable = 256;
    2071             : 
    2072             : // Adjust the function prologue to enable split stacks. This currently only
    2073             : // supports android and linux.
    2074             : //
    2075             : // The ABI of the segmented stack prologue is a little arbitrarily chosen, but
    2076             : // must be well defined in order to allow for consistent implementations of the
    2077             : // __morestack helper function. The ABI is also not a normal ABI in that it
    2078             : // doesn't follow the normal calling conventions because this allows the
    2079             : // prologue of each function to be optimized further.
    2080             : //
    2081             : // Currently, the ABI looks like (when calling __morestack)
    2082             : //
    2083             : //  * r4 holds the minimum stack size requested for this function call
    2084             : //  * r5 holds the stack size of the arguments to the function
    2085             : //  * the beginning of the function is 3 instructions after the call to
    2086             : //    __morestack
    2087             : //
    2088             : // Implementations of __morestack should use r4 to allocate a new stack, r5 to
    2089             : // place the arguments on to the new stack, and the 3-instruction knowledge to
    2090             : // jump directly to the body of the function when working on the new stack.
    2091             : //
    2092             : // An old (and possibly no longer compatible) implementation of __morestack for
    2093             : // ARM can be found at [1].
    2094             : //
    2095             : // [1] - https://github.com/mozilla/rust/blob/86efd9/src/rt/arch/arm/morestack.S
    2096          60 : void ARMFrameLowering::adjustForSegmentedStacks(
    2097             :     MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
    2098             :   unsigned Opcode;
    2099             :   unsigned CFIIndex;
    2100          60 :   const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
    2101          60 :   bool Thumb = ST->isThumb();
    2102             : 
    2103             :   // Sadly, this currently doesn't support varargs, platforms other than
    2104             :   // android/linux. Note that thumb1/thumb2 are support for android/linux.
    2105         120 :   if (MF.getFunction()->isVarArg())
    2106           0 :     report_fatal_error("Segmented stacks do not support vararg functions.");
    2107          90 :   if (!ST->isTargetAndroid() && !ST->isTargetLinux())
    2108           0 :     report_fatal_error("Segmented stacks not supported on this platform.");
    2109             : 
    2110          60 :   MachineFrameInfo &MFI = MF.getFrameInfo();
    2111          60 :   MachineModuleInfo &MMI = MF.getMMI();
    2112          60 :   MCContext &Context = MMI.getContext();
    2113          60 :   const MCRegisterInfo *MRI = Context.getRegisterInfo();
    2114             :   const ARMBaseInstrInfo &TII =
    2115          60 :       *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
    2116          60 :   ARMFunctionInfo *ARMFI = MF.getInfo<ARMFunctionInfo>();
    2117         112 :   DebugLoc DL;
    2118             : 
    2119          60 :   uint64_t StackSize = MFI.getStackSize();
    2120             : 
    2121             :   // Do not generate a prologue for functions with a stack of size zero
    2122          60 :   if (StackSize == 0)
    2123           8 :     return;
    2124             : 
    2125             :   // Use R4 and R5 as scratch registers.
    2126             :   // We save R4 and R5 before use and restore them before leaving the function.
    2127          52 :   unsigned ScratchReg0 = ARM::R4;
    2128          52 :   unsigned ScratchReg1 = ARM::R5;
    2129             :   uint64_t AlignedStackSize;
    2130             : 
    2131          52 :   MachineBasicBlock *PrevStackMBB = MF.CreateMachineBasicBlock();
    2132          52 :   MachineBasicBlock *PostStackMBB = MF.CreateMachineBasicBlock();
    2133          52 :   MachineBasicBlock *AllocMBB = MF.CreateMachineBasicBlock();
    2134          52 :   MachineBasicBlock *GetMBB = MF.CreateMachineBasicBlock();
    2135          52 :   MachineBasicBlock *McrMBB = MF.CreateMachineBasicBlock();
    2136             : 
    2137             :   // Grab everything that reaches PrologueMBB to update there liveness as well.
    2138         104 :   SmallPtrSet<MachineBasicBlock *, 8> BeforePrologueRegion;
    2139         104 :   SmallVector<MachineBasicBlock *, 2> WalkList;
    2140          52 :   WalkList.push_back(&PrologueMBB);
    2141             : 
    2142             :   do {
    2143          52 :     MachineBasicBlock *CurMBB = WalkList.pop_back_val();
    2144         104 :     for (MachineBasicBlock *PredBB : CurMBB->predecessors()) {
    2145           0 :       if (BeforePrologueRegion.insert(PredBB).second)
    2146           0 :         WalkList.push_back(PredBB);
    2147             :     }
    2148          52 :   } while (!WalkList.empty());
    2149             : 
    2150             :   // The order in that list is important.
    2151             :   // The blocks will all be inserted before PrologueMBB using that order.
    2152             :   // Therefore the block that should appear first in the CFG should appear
    2153             :   // first in the list.
    2154             :   MachineBasicBlock *AddedBlocks[] = {PrevStackMBB, McrMBB, GetMBB, AllocMBB,
    2155          52 :                                       PostStackMBB};
    2156             : 
    2157         312 :   for (MachineBasicBlock *B : AddedBlocks)
    2158         260 :     BeforePrologueRegion.insert(B);
    2159             : 
    2160         268 :   for (const auto &LI : PrologueMBB.liveins()) {
    2161         984 :     for (MachineBasicBlock *PredBB : BeforePrologueRegion)
    2162         820 :       PredBB->addLiveIn(LI);
    2163             :   }
    2164             : 
    2165             :   // Remove the newly added blocks from the list, since we know
    2166             :   // we do not have to do the following updates for them.
    2167         572 :   for (MachineBasicBlock *B : AddedBlocks) {
    2168         260 :     BeforePrologueRegion.erase(B);
    2169         780 :     MF.insert(PrologueMBB.getIterator(), B);
    2170             :   }
    2171             : 
    2172          52 :   for (MachineBasicBlock *MBB : BeforePrologueRegion) {
    2173             :     // Make sure the LiveIns are still sorted and unique.
    2174           0 :     MBB->sortUniqueLiveIns();
    2175             :     // Replace the edges to PrologueMBB by edges to the sequences
    2176             :     // we are about to add.
    2177           0 :     MBB->ReplaceUsesOfBlockWith(&PrologueMBB, AddedBlocks[0]);
    2178             :   }
    2179             : 
    2180             :   // The required stack size that is aligned to ARM constant criterion.
    2181          52 :   AlignedStackSize = alignToARMConstant(StackSize);
    2182             : 
    2183             :   // When the frame size is less than 256 we just compare the stack
    2184             :   // boundary directly to the value of the stack pointer, per gcc.
    2185          52 :   bool CompareStackPointer = AlignedStackSize < kSplitStackAvailable;
    2186             : 
    2187             :   // We will use two of the callee save registers as scratch registers so we
    2188             :   // need to save those registers onto the stack.
    2189             :   // We will use SR0 to hold stack limit and SR1 to hold the stack size
    2190             :   // requested and arguments for __morestack().
    2191             :   // SR0: Scratch Register #0
    2192             :   // SR1: Scratch Register #1
    2193             :   // push {SR0, SR1}
    2194          52 :   if (Thumb) {
    2195         104 :     BuildMI(PrevStackMBB, DL, TII.get(ARM::tPUSH))
    2196         104 :         .add(predOps(ARMCC::AL))
    2197          26 :         .addReg(ScratchReg0)
    2198          26 :         .addReg(ScratchReg1);
    2199             :   } else {
    2200         104 :     BuildMI(PrevStackMBB, DL, TII.get(ARM::STMDB_UPD))
    2201          26 :         .addReg(ARM::SP, RegState::Define)
    2202          26 :         .addReg(ARM::SP)
    2203         104 :         .add(predOps(ARMCC::AL))
    2204          26 :         .addReg(ScratchReg0)
    2205          26 :         .addReg(ScratchReg1);
    2206             :   }
    2207             : 
    2208             :   // Emit the relevant DWARF information about the change in stack pointer as
    2209             :   // well as where to find both r4 and r5 (the callee-save registers)
    2210          52 :   CFIIndex =
    2211         104 :       MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, -8));
    2212         156 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2213          52 :       .addCFIIndex(CFIIndex);
    2214         208 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2215          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg1, true), -4));
    2216         156 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2217          52 :       .addCFIIndex(CFIIndex);
    2218         208 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2219          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg0, true), -8));
    2220         156 :   BuildMI(PrevStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2221          52 :       .addCFIIndex(CFIIndex);
    2222             : 
    2223             :   // mov SR1, sp
    2224          52 :   if (Thumb) {
    2225         104 :     BuildMI(McrMBB, DL, TII.get(ARM::tMOVr), ScratchReg1)
    2226          26 :         .addReg(ARM::SP)
    2227          78 :         .add(predOps(ARMCC::AL));
    2228          26 :   } else if (CompareStackPointer) {
    2229          72 :     BuildMI(McrMBB, DL, TII.get(ARM::MOVr), ScratchReg1)
    2230          18 :         .addReg(ARM::SP)
    2231          72 :         .add(predOps(ARMCC::AL))
    2232          36 :         .add(condCodeOp());
    2233             :   }
    2234             : 
    2235             :   // sub SR1, sp, #StackSize
    2236          52 :   if (!CompareStackPointer && Thumb) {
    2237          32 :     BuildMI(McrMBB, DL, TII.get(ARM::tSUBi8), ScratchReg1)
    2238          24 :         .add(condCodeOp())
    2239           8 :         .addReg(ScratchReg1)
    2240          16 :         .addImm(AlignedStackSize)
    2241          24 :         .add(predOps(ARMCC::AL));
    2242          44 :   } else if (!CompareStackPointer) {
    2243          32 :     BuildMI(McrMBB, DL, TII.get(ARM::SUBri), ScratchReg1)
    2244           8 :         .addReg(ARM::SP)
    2245          16 :         .addImm(AlignedStackSize)
    2246          32 :         .add(predOps(ARMCC::AL))
    2247          16 :         .add(condCodeOp());
    2248             :   }
    2249             : 
    2250          52 :   if (Thumb && ST->isThumb1Only()) {
    2251          48 :     unsigned PCLabelId = ARMFI->createPICLabelUId();
    2252          48 :     ARMConstantPoolValue *NewCPV = ARMConstantPoolSymbol::Create(
    2253          24 :         MF.getFunction()->getContext(), "__STACK_LIMIT", PCLabelId, 0);
    2254          24 :     MachineConstantPool *MCP = MF.getConstantPool();
    2255          24 :     unsigned CPI = MCP->getConstantPoolIndex(NewCPV, 4);
    2256             : 
    2257             :     // ldr SR0, [pc, offset(STACK_LIMIT)]
    2258          72 :     BuildMI(GetMBB, DL, TII.get(ARM::tLDRpci), ScratchReg0)
    2259          24 :         .addConstantPoolIndex(CPI)
    2260          72 :         .add(predOps(ARMCC::AL));
    2261             : 
    2262             :     // ldr SR0, [SR0]
    2263          96 :     BuildMI(GetMBB, DL, TII.get(ARM::tLDRi), ScratchReg0)
    2264          24 :         .addReg(ScratchReg0)
    2265          24 :         .addImm(0)
    2266          72 :         .add(predOps(ARMCC::AL));
    2267             :   } else {
    2268             :     // Get TLS base address from the coprocessor
    2269             :     // mrc p15, #0, SR0, c13, c0, #3
    2270         112 :     BuildMI(McrMBB, DL, TII.get(ARM::MRC), ScratchReg0)
    2271          28 :         .addImm(15)
    2272          28 :         .addImm(0)
    2273          28 :         .addImm(13)
    2274          28 :         .addImm(0)
    2275          28 :         .addImm(3)
    2276          84 :         .add(predOps(ARMCC::AL));
    2277             : 
    2278             :     // Use the last tls slot on android and a private field of the TCP on linux.
    2279             :     assert(ST->isTargetAndroid() || ST->isTargetLinux());
    2280          28 :     unsigned TlsOffset = ST->isTargetAndroid() ? 63 : 1;
    2281             : 
    2282             :     // Get the stack limit from the right offset
    2283             :     // ldr SR0, [sr0, #4 * TlsOffset]
    2284         112 :     BuildMI(GetMBB, DL, TII.get(ARM::LDRi12), ScratchReg0)
    2285          28 :         .addReg(ScratchReg0)
    2286          56 :         .addImm(4 * TlsOffset)
    2287          84 :         .add(predOps(ARMCC::AL));
    2288             :   }
    2289             : 
    2290             :   // Compare stack limit with stack size requested.
    2291             :   // cmp SR0, SR1
    2292          52 :   Opcode = Thumb ? ARM::tCMPr : ARM::CMPrr;
    2293         208 :   BuildMI(GetMBB, DL, TII.get(Opcode))
    2294          52 :       .addReg(ScratchReg0)
    2295          52 :       .addReg(ScratchReg1)
    2296         156 :       .add(predOps(ARMCC::AL));
    2297             : 
    2298             :   // This jump is taken if StackLimit < SP - stack required.
    2299          52 :   Opcode = Thumb ? ARM::tBcc : ARM::Bcc;
    2300         260 :   BuildMI(GetMBB, DL, TII.get(Opcode)).addMBB(PostStackMBB)
    2301          52 :        .addImm(ARMCC::LO)
    2302          52 :        .addReg(ARM::CPSR);
    2303             : 
    2304             : 
    2305             :   // Calling __morestack(StackSize, Size of stack arguments).
    2306             :   // __morestack knows that the stack size requested is in SR0(r4)
    2307             :   // and amount size of stack arguments is in SR1(r5).
    2308             : 
    2309             :   // Pass first argument for the __morestack by Scratch Register #0.
    2310             :   //   The amount size of stack required
    2311          52 :   if (Thumb) {
    2312          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::tMOVi8), ScratchReg0)
    2313          78 :         .add(condCodeOp())
    2314          52 :         .addImm(AlignedStackSize)
    2315          78 :         .add(predOps(ARMCC::AL));
    2316             :   } else {
    2317          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::MOVi), ScratchReg0)
    2318          52 :         .addImm(AlignedStackSize)
    2319         104 :         .add(predOps(ARMCC::AL))
    2320          52 :         .add(condCodeOp());
    2321             :   }
    2322             :   // Pass second argument for the __morestack by Scratch Register #1.
    2323             :   //   The amount size of stack consumed to save function arguments.
    2324          52 :   if (Thumb) {
    2325          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::tMOVi8), ScratchReg1)
    2326          78 :         .add(condCodeOp())
    2327          52 :         .addImm(alignToARMConstant(ARMFI->getArgumentStackSize()))
    2328          78 :         .add(predOps(ARMCC::AL));
    2329             :   } else {
    2330          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::MOVi), ScratchReg1)
    2331          52 :         .addImm(alignToARMConstant(ARMFI->getArgumentStackSize()))
    2332         104 :         .add(predOps(ARMCC::AL))
    2333          52 :         .add(condCodeOp());
    2334             :   }
    2335             : 
    2336             :   // push {lr} - Save return address of this function.
    2337          52 :   if (Thumb) {
    2338         104 :     BuildMI(AllocMBB, DL, TII.get(ARM::tPUSH))
    2339         104 :         .add(predOps(ARMCC::AL))
    2340          26 :         .addReg(ARM::LR);
    2341             :   } else {
    2342         104 :     BuildMI(AllocMBB, DL, TII.get(ARM::STMDB_UPD))
    2343          26 :         .addReg(ARM::SP, RegState::Define)
    2344          26 :         .addReg(ARM::SP)
    2345         104 :         .add(predOps(ARMCC::AL))
    2346          26 :         .addReg(ARM::LR);
    2347             :   }
    2348             : 
    2349             :   // Emit the DWARF info about the change in stack as well as where to find the
    2350             :   // previous link register
    2351          52 :   CFIIndex =
    2352         104 :       MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, -12));
    2353         156 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2354          52 :       .addCFIIndex(CFIIndex);
    2355         208 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
    2356          52 :         nullptr, MRI->getDwarfRegNum(ARM::LR, true), -12));
    2357         156 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2358          52 :       .addCFIIndex(CFIIndex);
    2359             : 
    2360             :   // Call __morestack().
    2361          52 :   if (Thumb) {
    2362          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::tBL))
    2363         104 :         .add(predOps(ARMCC::AL))
    2364          26 :         .addExternalSymbol("__morestack");
    2365             :   } else {
    2366          78 :     BuildMI(AllocMBB, DL, TII.get(ARM::BL))
    2367          26 :         .addExternalSymbol("__morestack");
    2368             :   }
    2369             : 
    2370             :   // pop {lr} - Restore return address of this original function.
    2371          52 :   if (Thumb) {
    2372             :     if (ST->isThumb1Only()) {
    2373          96 :       BuildMI(AllocMBB, DL, TII.get(ARM::tPOP))
    2374          96 :           .add(predOps(ARMCC::AL))
    2375          24 :           .addReg(ScratchReg0);
    2376          96 :       BuildMI(AllocMBB, DL, TII.get(ARM::tMOVr), ARM::LR)
    2377          24 :           .addReg(ScratchReg0)
    2378          72 :           .add(predOps(ARMCC::AL));
    2379             :     } else {
    2380           8 :       BuildMI(AllocMBB, DL, TII.get(ARM::t2LDR_POST))
    2381           2 :           .addReg(ARM::LR, RegState::Define)
    2382           2 :           .addReg(ARM::SP, RegState::Define)
    2383           2 :           .addReg(ARM::SP)
    2384           2 :           .addImm(4)
    2385           6 :           .add(predOps(ARMCC::AL));
    2386             :     }
    2387             :   } else {
    2388         104 :     BuildMI(AllocMBB, DL, TII.get(ARM::LDMIA_UPD))
    2389          26 :         .addReg(ARM::SP, RegState::Define)
    2390          26 :         .addReg(ARM::SP)
    2391         104 :         .add(predOps(ARMCC::AL))
    2392          26 :         .addReg(ARM::LR);
    2393             :   }
    2394             : 
    2395             :   // Restore SR0 and SR1 in case of __morestack() was called.
    2396             :   // __morestack() will skip PostStackMBB block so we need to restore
    2397             :   // scratch registers from here.
    2398             :   // pop {SR0, SR1}
    2399          52 :   if (Thumb) {
    2400         104 :     BuildMI(AllocMBB, DL, TII.get(ARM::tPOP))
    2401         104 :         .add(predOps(ARMCC::AL))
    2402          26 :         .addReg(ScratchReg0)
    2403          26 :         .addReg(ScratchReg1);
    2404             :   } else {
    2405         104 :     BuildMI(AllocMBB, DL, TII.get(ARM::LDMIA_UPD))
    2406          26 :         .addReg(ARM::SP, RegState::Define)
    2407          26 :         .addReg(ARM::SP)
    2408         104 :         .add(predOps(ARMCC::AL))
    2409          26 :         .addReg(ScratchReg0)
    2410          26 :         .addReg(ScratchReg1);
    2411             :   }
    2412             : 
    2413             :   // Update the CFA offset now that we've popped
    2414         156 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, 0));
    2415         156 :   BuildMI(AllocMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2416          52 :       .addCFIIndex(CFIIndex);
    2417             : 
    2418             :   // Return from this function.
    2419         364 :   BuildMI(AllocMBB, DL, TII.get(ST->getReturnOpcode())).add(predOps(ARMCC::AL));
    2420             : 
    2421             :   // Restore SR0 and SR1 in case of __morestack() was not called.
    2422             :   // pop {SR0, SR1}
    2423          52 :   if (Thumb) {
    2424         104 :     BuildMI(PostStackMBB, DL, TII.get(ARM::tPOP))
    2425         104 :         .add(predOps(ARMCC::AL))
    2426          26 :         .addReg(ScratchReg0)
    2427          26 :         .addReg(ScratchReg1);
    2428             :   } else {
    2429         104 :     BuildMI(PostStackMBB, DL, TII.get(ARM::LDMIA_UPD))
    2430          26 :         .addReg(ARM::SP, RegState::Define)
    2431          26 :         .addReg(ARM::SP)
    2432         104 :         .add(predOps(ARMCC::AL))
    2433          26 :         .addReg(ScratchReg0)
    2434          26 :         .addReg(ScratchReg1);
    2435             :   }
    2436             : 
    2437             :   // Update the CFA offset now that we've popped
    2438         156 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(nullptr, 0));
    2439         156 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2440          52 :       .addCFIIndex(CFIIndex);
    2441             : 
    2442             :   // Tell debuggers that r4 and r5 are now the same as they were in the
    2443             :   // previous function, that they're the "Same Value".
    2444         208 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
    2445          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg0, true)));
    2446         156 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2447          52 :       .addCFIIndex(CFIIndex);
    2448         208 :   CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(
    2449          52 :       nullptr, MRI->getDwarfRegNum(ScratchReg1, true)));
    2450         156 :   BuildMI(PostStackMBB, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
    2451          52 :       .addCFIIndex(CFIIndex);
    2452             : 
    2453             :   // Organizing MBB lists
    2454          52 :   PostStackMBB->addSuccessor(&PrologueMBB);
    2455             : 
    2456          52 :   AllocMBB->addSuccessor(PostStackMBB);
    2457             : 
    2458          52 :   GetMBB->addSuccessor(PostStackMBB);
    2459          52 :   GetMBB->addSuccessor(AllocMBB);
    2460             : 
    2461          52 :   McrMBB->addSuccessor(GetMBB);
    2462             : 
    2463          52 :   PrevStackMBB->addSuccessor(McrMBB);
    2464             : 
    2465             : #ifdef EXPENSIVE_CHECKS
    2466             :   MF.verify();
    2467             : #endif
    2468      216918 : }

Generated by: LCOV version 1.13