Line data Source code
1 : //===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This pass is responsible for finalizing the functions frame layout, saving
11 : // callee saved registers, and for emitting prolog & epilog code for the
12 : // function.
13 : //
14 : // This pass must be run after register allocation. After this pass is
15 : // executed, it is illegal to construct MO_FrameIndex operands.
16 : //
17 : //===----------------------------------------------------------------------===//
18 :
19 : #include "llvm/ADT/ArrayRef.h"
20 : #include "llvm/ADT/BitVector.h"
21 : #include "llvm/ADT/DepthFirstIterator.h"
22 : #include "llvm/ADT/STLExtras.h"
23 : #include "llvm/ADT/SetVector.h"
24 : #include "llvm/ADT/SmallPtrSet.h"
25 : #include "llvm/ADT/SmallSet.h"
26 : #include "llvm/ADT/SmallVector.h"
27 : #include "llvm/ADT/Statistic.h"
28 : #include "llvm/Analysis/OptimizationRemarkEmitter.h"
29 : #include "llvm/CodeGen/MachineBasicBlock.h"
30 : #include "llvm/CodeGen/MachineDominators.h"
31 : #include "llvm/CodeGen/MachineFrameInfo.h"
32 : #include "llvm/CodeGen/MachineFunction.h"
33 : #include "llvm/CodeGen/MachineFunctionPass.h"
34 : #include "llvm/CodeGen/MachineInstr.h"
35 : #include "llvm/CodeGen/MachineLoopInfo.h"
36 : #include "llvm/CodeGen/MachineModuleInfo.h"
37 : #include "llvm/CodeGen/MachineOperand.h"
38 : #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
39 : #include "llvm/CodeGen/MachineRegisterInfo.h"
40 : #include "llvm/CodeGen/RegisterScavenging.h"
41 : #include "llvm/CodeGen/TargetFrameLowering.h"
42 : #include "llvm/CodeGen/TargetInstrInfo.h"
43 : #include "llvm/CodeGen/TargetOpcodes.h"
44 : #include "llvm/CodeGen/TargetRegisterInfo.h"
45 : #include "llvm/CodeGen/TargetSubtargetInfo.h"
46 : #include "llvm/CodeGen/WinEHFuncInfo.h"
47 : #include "llvm/IR/Attributes.h"
48 : #include "llvm/IR/CallingConv.h"
49 : #include "llvm/IR/DebugInfoMetadata.h"
50 : #include "llvm/IR/DiagnosticInfo.h"
51 : #include "llvm/IR/Function.h"
52 : #include "llvm/IR/InlineAsm.h"
53 : #include "llvm/IR/LLVMContext.h"
54 : #include "llvm/MC/MCRegisterInfo.h"
55 : #include "llvm/Pass.h"
56 : #include "llvm/Support/CodeGen.h"
57 : #include "llvm/Support/CommandLine.h"
58 : #include "llvm/Support/Debug.h"
59 : #include "llvm/Support/ErrorHandling.h"
60 : #include "llvm/Support/MathExtras.h"
61 : #include "llvm/Support/raw_ostream.h"
62 : #include "llvm/Target/TargetMachine.h"
63 : #include "llvm/Target/TargetOptions.h"
64 : #include <algorithm>
65 : #include <cassert>
66 : #include <cstdint>
67 : #include <functional>
68 : #include <limits>
69 : #include <utility>
70 : #include <vector>
71 :
72 : using namespace llvm;
73 :
74 : #define DEBUG_TYPE "prologepilog"
75 :
76 : using MBBVector = SmallVector<MachineBasicBlock *, 4>;
77 :
78 : namespace {
79 :
80 : class PEI : public MachineFunctionPass {
81 : public:
82 : static char ID;
83 :
84 27224 : PEI() : MachineFunctionPass(ID) {
85 27224 : initializePEIPass(*PassRegistry::getPassRegistry());
86 27224 : }
87 :
88 : void getAnalysisUsage(AnalysisUsage &AU) const override;
89 :
90 : /// runOnMachineFunction - Insert prolog/epilog code and replace abstract
91 : /// frame indexes with appropriate references.
92 : bool runOnMachineFunction(MachineFunction &MF) override;
93 :
94 : private:
95 : RegScavenger *RS;
96 :
97 : // MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved
98 : // stack frame indexes.
99 : unsigned MinCSFrameIndex = std::numeric_limits<unsigned>::max();
100 : unsigned MaxCSFrameIndex = 0;
101 :
102 : // Save and Restore blocks of the current function. Typically there is a
103 : // single save block, unless Windows EH funclets are involved.
104 : MBBVector SaveBlocks;
105 : MBBVector RestoreBlocks;
106 :
107 : // Flag to control whether to use the register scavenger to resolve
108 : // frame index materialization registers. Set according to
109 : // TRI->requiresFrameIndexScavenging() for the current function.
110 : bool FrameIndexVirtualScavenging;
111 :
112 : // Flag to control whether the scavenger should be passed even though
113 : // FrameIndexVirtualScavenging is used.
114 : bool FrameIndexEliminationScavenging;
115 :
116 : // Emit remarks.
117 : MachineOptimizationRemarkEmitter *ORE = nullptr;
118 :
119 : void calculateCallFrameInfo(MachineFunction &MF);
120 : void calculateSaveRestoreBlocks(MachineFunction &MF);
121 : void spillCalleeSavedRegs(MachineFunction &MF);
122 :
123 : void calculateFrameObjectOffsets(MachineFunction &MF);
124 : void replaceFrameIndices(MachineFunction &MF);
125 : void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
126 : int &SPAdj);
127 : void insertPrologEpilogCode(MachineFunction &MF);
128 : };
129 :
130 : } // end anonymous namespace
131 :
132 : char PEI::ID = 0;
133 :
134 : char &llvm::PrologEpilogCodeInserterID = PEI::ID;
135 :
136 : static cl::opt<unsigned>
137 : WarnStackSize("warn-stack-size", cl::Hidden, cl::init((unsigned)-1),
138 : cl::desc("Warn for stack size bigger than the given"
139 : " number"));
140 :
141 31780 : INITIALIZE_PASS_BEGIN(PEI, DEBUG_TYPE, "Prologue/Epilogue Insertion", false,
142 : false)
143 31780 : INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
144 31780 : INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
145 31780 : INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
146 112371 : INITIALIZE_PASS_END(PEI, DEBUG_TYPE,
147 : "Prologue/Epilogue Insertion & Frame Finalization", false,
148 : false)
149 :
150 27207 : MachineFunctionPass *llvm::createPrologEpilogInserterPass() {
151 27207 : return new PEI();
152 : }
153 :
154 : STATISTIC(NumBytesStackSpace,
155 : "Number of bytes used for stack in all functions");
156 :
157 27033 : void PEI::getAnalysisUsage(AnalysisUsage &AU) const {
158 27033 : AU.setPreservesCFG();
159 : AU.addPreserved<MachineLoopInfo>();
160 : AU.addPreserved<MachineDominatorTree>();
161 : AU.addRequired<MachineOptimizationRemarkEmitterPass>();
162 27033 : MachineFunctionPass::getAnalysisUsage(AU);
163 27033 : }
164 :
165 : /// StackObjSet - A set of stack object indexes
166 : using StackObjSet = SmallSetVector<int, 8>;
167 :
168 : /// runOnMachineFunction - Insert prolog/epilog code and replace abstract
169 : /// frame indexes with appropriate references.
170 404392 : bool PEI::runOnMachineFunction(MachineFunction &MF) {
171 404392 : const Function &F = MF.getFunction();
172 404392 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
173 404392 : const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
174 :
175 404392 : RS = TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr;
176 404392 : FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(MF);
177 808021 : FrameIndexEliminationScavenging = (RS && !FrameIndexVirtualScavenging) ||
178 403629 : TRI->requiresFrameIndexReplacementScavenging(MF);
179 404392 : ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
180 :
181 : // Calculate the MaxCallFrameSize and AdjustsStack variables for the
182 : // function's frame information. Also eliminates call frame pseudo
183 : // instructions.
184 404392 : calculateCallFrameInfo(MF);
185 :
186 : // Determine placement of CSR spill/restore code and prolog/epilog code:
187 : // place all spills in the entry block, all restores in return blocks.
188 404392 : calculateSaveRestoreBlocks(MF);
189 :
190 : // Handle CSR spilling and restoring, for targets that need it.
191 404391 : if (MF.getTarget().usesPhysRegsForPEI())
192 401409 : spillCalleeSavedRegs(MF);
193 :
194 : // Allow the target machine to make final modifications to the function
195 : // before the frame layout is finalized.
196 404392 : TFI->processFunctionBeforeFrameFinalized(MF, RS);
197 :
198 : // Calculate actual frame offsets for all abstract stack objects...
199 404392 : calculateFrameObjectOffsets(MF);
200 :
201 : // Add prolog and epilog code to the function. This function is required
202 : // to align the stack frame as necessary for any stack variables or
203 : // called functions. Because of this, calculateCalleeSavedRegisters()
204 : // must be called before this function in order to set the AdjustsStack
205 : // and MaxCallFrameSize variables.
206 404392 : if (!F.hasFnAttribute(Attribute::Naked))
207 404310 : insertPrologEpilogCode(MF);
208 :
209 : // Replace all MO_FrameIndex operands with physical register references
210 : // and actual offsets.
211 : //
212 404385 : replaceFrameIndices(MF);
213 :
214 : // If register scavenging is needed, as we've enabled doing it as a
215 : // post-pass, scavenge the virtual registers that frame index elimination
216 : // inserted.
217 404385 : if (TRI->requiresRegisterScavenging(MF) && FrameIndexVirtualScavenging)
218 65567 : scavengeFrameVirtualRegs(MF, *RS);
219 :
220 : // Warn on stack size when we exceeds the given limit.
221 404385 : MachineFrameInfo &MFI = MF.getFrameInfo();
222 404385 : uint64_t StackSize = MFI.getStackSize();
223 404385 : if (WarnStackSize.getNumOccurrences() > 0 && WarnStackSize < StackSize) {
224 : DiagnosticInfoStackSize DiagStackSize(F, StackSize);
225 20 : F.getContext().diagnose(DiagStackSize);
226 : }
227 404385 : ORE->emit([&]() {
228 : return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "StackSize",
229 : MF.getFunction().getSubprogram(),
230 : &MF.front())
231 : << ore::NV("NumStackBytes", StackSize) << " stack bytes in function";
232 : });
233 :
234 404385 : delete RS;
235 : SaveBlocks.clear();
236 : RestoreBlocks.clear();
237 : MFI.setSavePoint(nullptr);
238 : MFI.setRestorePoint(nullptr);
239 404385 : return true;
240 : }
241 :
242 : /// Calculate the MaxCallFrameSize and AdjustsStack
243 : /// variables for the function's frame information and eliminate call frame
244 : /// pseudo instructions.
245 0 : void PEI::calculateCallFrameInfo(MachineFunction &MF) {
246 0 : const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
247 0 : const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
248 0 : MachineFrameInfo &MFI = MF.getFrameInfo();
249 :
250 : unsigned MaxCallFrameSize = 0;
251 0 : bool AdjustsStack = MFI.adjustsStack();
252 :
253 : // Get the function call frame set-up and tear-down instruction opcode
254 0 : unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode();
255 0 : unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
256 :
257 : // Early exit for targets which have no call frame setup/destroy pseudo
258 : // instructions.
259 0 : if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
260 0 : return;
261 :
262 : std::vector<MachineBasicBlock::iterator> FrameSDOps;
263 0 : for (MachineFunction::iterator BB = MF.begin(), E = MF.end(); BB != E; ++BB)
264 0 : for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ++I)
265 : if (TII.isFrameInstr(*I)) {
266 0 : unsigned Size = TII.getFrameSize(*I);
267 0 : if (Size > MaxCallFrameSize) MaxCallFrameSize = Size;
268 : AdjustsStack = true;
269 0 : FrameSDOps.push_back(I);
270 0 : } else if (I->isInlineAsm()) {
271 : // Some inline asm's need a stack frame, as indicated by operand 1.
272 0 : unsigned ExtraInfo = I->getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
273 0 : if (ExtraInfo & InlineAsm::Extra_IsAlignStack)
274 : AdjustsStack = true;
275 : }
276 :
277 : assert(!MFI.isMaxCallFrameSizeComputed() ||
278 : (MFI.getMaxCallFrameSize() == MaxCallFrameSize &&
279 : MFI.adjustsStack() == AdjustsStack));
280 : MFI.setAdjustsStack(AdjustsStack);
281 : MFI.setMaxCallFrameSize(MaxCallFrameSize);
282 :
283 : for (std::vector<MachineBasicBlock::iterator>::iterator
284 0 : i = FrameSDOps.begin(), e = FrameSDOps.end(); i != e; ++i) {
285 0 : MachineBasicBlock::iterator I = *i;
286 :
287 : // If call frames are not being included as part of the stack frame, and
288 : // the target doesn't indicate otherwise, remove the call frame pseudos
289 : // here. The sub/add sp instruction pairs are still inserted, but we don't
290 : // need to track the SP adjustment for frame index elimination.
291 0 : if (TFI->canSimplifyCallFramePseudos(MF))
292 0 : TFI->eliminateCallFramePseudoInstr(MF, *I->getParent(), I);
293 : }
294 : }
295 :
296 : /// Compute the sets of entry and return blocks for saving and restoring
297 : /// callee-saved registers, and placing prolog and epilog code.
298 404392 : void PEI::calculateSaveRestoreBlocks(MachineFunction &MF) {
299 404392 : const MachineFrameInfo &MFI = MF.getFrameInfo();
300 :
301 : // Even when we do not change any CSR, we still want to insert the
302 : // prologue and epilogue of the function.
303 : // So set the save points for those.
304 :
305 : // Use the points found by shrink-wrapping, if any.
306 404392 : if (MFI.getSavePoint()) {
307 564 : SaveBlocks.push_back(MFI.getSavePoint());
308 : assert(MFI.getRestorePoint() && "Both restore and save must be set");
309 564 : MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
310 : // If RestoreBlock does not have any successor and is not a return block
311 : // then the end point is unreachable and we do not need to insert any
312 : // epilogue.
313 564 : if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
314 502 : RestoreBlocks.push_back(RestoreBlock);
315 : return;
316 : }
317 :
318 : // Save refs to entry and return blocks.
319 807656 : SaveBlocks.push_back(&MF.front());
320 3685025 : for (MachineBasicBlock &MBB : MF) {
321 3281197 : if (MBB.isEHFuncletEntry())
322 118 : SaveBlocks.push_back(&MBB);
323 3281197 : if (MBB.isReturnBlock())
324 402672 : RestoreBlocks.push_back(&MBB);
325 : }
326 : }
327 :
328 401409 : static void assignCalleeSavedSpillSlots(MachineFunction &F,
329 : const BitVector &SavedRegs,
330 : unsigned &MinCSFrameIndex,
331 : unsigned &MaxCSFrameIndex) {
332 401409 : if (SavedRegs.empty())
333 73700 : return;
334 :
335 401403 : const TargetRegisterInfo *RegInfo = F.getSubtarget().getRegisterInfo();
336 401403 : const MCPhysReg *CSRegs = F.getRegInfo().getCalleeSavedRegs();
337 :
338 : std::vector<CalleeSavedInfo> CSI;
339 4211130 : for (unsigned i = 0; CSRegs[i]; ++i) {
340 3809727 : unsigned Reg = CSRegs[i];
341 3809727 : if (SavedRegs.test(Reg))
342 61883 : CSI.push_back(CalleeSavedInfo(Reg));
343 : }
344 :
345 401403 : const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering();
346 401403 : MachineFrameInfo &MFI = F.getFrameInfo();
347 401403 : if (!TFI->assignCalleeSavedSpillSlots(F, RegInfo, CSI)) {
348 : // If target doesn't implement this, use generic code.
349 :
350 82654 : if (CSI.empty())
351 73694 : return; // Early exit if no callee saved registers are modified!
352 :
353 : unsigned NumFixedSpillSlots;
354 : const TargetFrameLowering::SpillSlot *FixedSpillSlots =
355 8960 : TFI->getCalleeSavedSpillSlots(NumFixedSpillSlots);
356 :
357 : // Now that we know which registers need to be saved and restored, allocate
358 : // stack slots for them.
359 36826 : for (auto &CS : CSI) {
360 27866 : unsigned Reg = CS.getReg();
361 27866 : const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
362 :
363 : int FrameIdx;
364 27866 : if (RegInfo->hasReservedSpillSlot(F, Reg, FrameIdx)) {
365 78 : CS.setFrameIdx(FrameIdx);
366 78 : continue;
367 : }
368 :
369 : // Check to see if this physreg must be spilled to a particular stack slot
370 : // on this target.
371 : const TargetFrameLowering::SpillSlot *FixedSlot = FixedSpillSlots;
372 125778 : while (FixedSlot != FixedSpillSlots + NumFixedSpillSlots &&
373 102767 : FixedSlot->Reg != Reg)
374 97990 : ++FixedSlot;
375 :
376 : unsigned Size = RegInfo->getSpillSize(*RC);
377 27788 : if (FixedSlot == FixedSpillSlots + NumFixedSpillSlots) {
378 : // Nope, just spill it anywhere convenient.
379 23011 : unsigned Align = RegInfo->getSpillAlignment(*RC);
380 23011 : unsigned StackAlign = TFI->getStackAlignment();
381 :
382 : // We may not be able to satisfy the desired alignment specification of
383 : // the TargetRegisterClass if the stack alignment is smaller. Use the
384 : // min.
385 23011 : Align = std::min(Align, StackAlign);
386 23011 : FrameIdx = MFI.CreateStackObject(Size, Align, true);
387 23011 : if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
388 23011 : if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
389 : } else {
390 : // Spill it to the stack where we must.
391 4777 : FrameIdx = MFI.CreateFixedSpillStackObject(Size, FixedSlot->Offset);
392 : }
393 :
394 27788 : CS.setFrameIdx(FrameIdx);
395 : }
396 : }
397 :
398 : MFI.setCalleeSavedInfo(CSI);
399 : }
400 :
401 : /// Helper function to update the liveness information for the callee-saved
402 : /// registers.
403 21908 : static void updateLiveness(MachineFunction &MF) {
404 21908 : MachineFrameInfo &MFI = MF.getFrameInfo();
405 : // Visited will contain all the basic blocks that are in the region
406 : // where the callee saved registers are alive:
407 : // - Anything that is not Save or Restore -> LiveThrough.
408 : // - Save -> LiveIn.
409 : // - Restore -> LiveOut.
410 : // The live-out is not attached to the block, so no need to keep
411 : // Restore in this set.
412 : SmallPtrSet<MachineBasicBlock *, 8> Visited;
413 : SmallVector<MachineBasicBlock *, 8> WorkList;
414 21908 : MachineBasicBlock *Entry = &MF.front();
415 21908 : MachineBasicBlock *Save = MFI.getSavePoint();
416 :
417 21908 : if (!Save)
418 : Save = Entry;
419 :
420 21908 : if (Entry != Save) {
421 310 : WorkList.push_back(Entry);
422 310 : Visited.insert(Entry);
423 : }
424 21908 : Visited.insert(Save);
425 :
426 21908 : MachineBasicBlock *Restore = MFI.getRestorePoint();
427 21908 : if (Restore)
428 : // By construction Restore cannot be visited, otherwise it
429 : // means there exists a path to Restore that does not go
430 : // through Save.
431 310 : WorkList.push_back(Restore);
432 :
433 23063 : while (!WorkList.empty()) {
434 : const MachineBasicBlock *CurBB = WorkList.pop_back_val();
435 : // By construction, the region that is after the save point is
436 : // dominated by the Save and post-dominated by the Restore.
437 1155 : if (CurBB == Save && Save != Restore)
438 : continue;
439 : // Enqueue all the successors not already visited.
440 : // Those are by construction either before Save or after Restore.
441 2271 : for (MachineBasicBlock *SuccBB : CurBB->successors())
442 1116 : if (Visited.insert(SuccBB).second)
443 535 : WorkList.push_back(SuccBB);
444 : }
445 :
446 : const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
447 :
448 21908 : MachineRegisterInfo &MRI = MF.getRegInfo();
449 105536 : for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
450 125204 : for (MachineBasicBlock *MBB : Visited) {
451 126968 : MCPhysReg Reg = CSI[i].getReg();
452 : // Add the callee-saved register as live-in.
453 : // It's killed at the spill.
454 63484 : if (!MRI.isReserved(Reg) && !MBB->isLiveIn(Reg))
455 : MBB->addLiveIn(Reg);
456 : }
457 : }
458 21908 : }
459 :
460 : /// Insert restore code for the callee-saved registers used in the function.
461 21908 : static void insertCSRSaves(MachineBasicBlock &SaveBlock,
462 : ArrayRef<CalleeSavedInfo> CSI) {
463 21908 : MachineFunction &MF = *SaveBlock.getParent();
464 21908 : const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
465 21908 : const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
466 21908 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
467 :
468 21908 : MachineBasicBlock::iterator I = SaveBlock.begin();
469 43816 : if (!TFI->spillCalleeSavedRegisters(SaveBlock, I, CSI, TRI)) {
470 675 : for (const CalleeSavedInfo &CS : CSI) {
471 : // Insert the spill to the stack frame.
472 548 : unsigned Reg = CS.getReg();
473 548 : const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
474 548 : TII.storeRegToStackSlot(SaveBlock, I, Reg, true, CS.getFrameIdx(), RC,
475 548 : TRI);
476 : }
477 : }
478 21908 : }
479 :
480 : /// Insert restore code for the callee-saved registers used in the function.
481 22215 : static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
482 : std::vector<CalleeSavedInfo> &CSI) {
483 22215 : MachineFunction &MF = *RestoreBlock.getParent();
484 22215 : const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
485 22215 : const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
486 22215 : const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
487 :
488 : // Restore all registers immediately before the return and any
489 : // terminators that precede it.
490 22215 : MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
491 :
492 22215 : if (!TFI->restoreCalleeSavedRegisters(RestoreBlock, I, CSI, TRI)) {
493 6314 : for (const CalleeSavedInfo &CI : reverse(CSI)) {
494 4214 : unsigned Reg = CI.getReg();
495 4214 : const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
496 4214 : TII.loadRegFromStackSlot(RestoreBlock, I, Reg, CI.getFrameIdx(), RC, TRI);
497 : assert(I != RestoreBlock.begin() &&
498 : "loadRegFromStackSlot didn't insert any code!");
499 : // Insert in reverse order. loadRegFromStackSlot can insert
500 : // multiple instructions.
501 : }
502 : }
503 22215 : }
504 :
505 401409 : void PEI::spillCalleeSavedRegs(MachineFunction &MF) {
506 : // We can't list this requirement in getRequiredProperties because some
507 : // targets (WebAssembly) use virtual registers past this point, and the pass
508 : // pipeline is set up without giving the passes a chance to look at the
509 : // TargetMachine.
510 : // FIXME: Find a way to express this in getRequiredProperties.
511 : assert(MF.getProperties().hasProperty(
512 : MachineFunctionProperties::Property::NoVRegs));
513 :
514 401409 : const Function &F = MF.getFunction();
515 401409 : const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
516 401409 : MachineFrameInfo &MFI = MF.getFrameInfo();
517 401409 : MinCSFrameIndex = std::numeric_limits<unsigned>::max();
518 401409 : MaxCSFrameIndex = 0;
519 :
520 : // Determine which of the registers in the callee save list should be saved.
521 : BitVector SavedRegs;
522 401409 : TFI->determineCalleeSaves(MF, SavedRegs, RS);
523 :
524 : // Assign stack slots for any callee-saved registers that must be spilled.
525 401409 : assignCalleeSavedSpillSlots(MF, SavedRegs, MinCSFrameIndex, MaxCSFrameIndex);
526 :
527 : // Add the code to save and restore the callee saved registers.
528 401409 : if (!F.hasFnAttribute(Attribute::Naked)) {
529 : MFI.setCalleeSavedInfoValid(true);
530 :
531 : std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
532 401327 : if (!CSI.empty()) {
533 43769 : for (MachineBasicBlock *SaveBlock : SaveBlocks) {
534 21908 : insertCSRSaves(*SaveBlock, CSI);
535 : // Update the live-in information of all the blocks up to the save
536 : // point.
537 21908 : updateLiveness(MF);
538 : }
539 44076 : for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
540 22215 : insertCSRRestores(*RestoreBlock, CSI);
541 : }
542 : }
543 401409 : }
544 :
545 : /// AdjustStackOffset - Helper function used to adjust the stack frame offset.
546 : static inline void
547 7709914 : AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx,
548 : bool StackGrowsDown, int64_t &Offset,
549 : unsigned &MaxAlign, unsigned Skew) {
550 : // If the stack grows down, add the object size to find the lowest address.
551 7709914 : if (StackGrowsDown)
552 7706575 : Offset += MFI.getObjectSize(FrameIdx);
553 :
554 7709914 : unsigned Align = MFI.getObjectAlignment(FrameIdx);
555 :
556 : // If the alignment of this object is greater than that of the stack, then
557 : // increase the stack alignment to match.
558 7709914 : MaxAlign = std::max(MaxAlign, Align);
559 :
560 : // Adjust to alignment boundary.
561 7709914 : Offset = alignTo(Offset, Align, Skew);
562 :
563 7709914 : if (StackGrowsDown) {
564 : LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << -Offset
565 : << "]\n");
566 7706575 : MFI.setObjectOffset(FrameIdx, -Offset); // Set the computed offset
567 : } else {
568 : LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << Offset
569 : << "]\n");
570 : MFI.setObjectOffset(FrameIdx, Offset);
571 3339 : Offset += MFI.getObjectSize(FrameIdx);
572 : }
573 7709914 : }
574 :
575 : /// Compute which bytes of fixed and callee-save stack area are unused and keep
576 : /// track of them in StackBytesFree.
577 : static inline void
578 173 : computeFreeStackSlots(MachineFrameInfo &MFI, bool StackGrowsDown,
579 : unsigned MinCSFrameIndex, unsigned MaxCSFrameIndex,
580 : int64_t FixedCSEnd, BitVector &StackBytesFree) {
581 : // Avoid undefined int64_t -> int conversion below in extreme case.
582 173 : if (FixedCSEnd > std::numeric_limits<int>::max())
583 0 : return;
584 :
585 173 : StackBytesFree.resize(FixedCSEnd, true);
586 :
587 : SmallVector<int, 16> AllocatedFrameSlots;
588 : // Add fixed objects.
589 381 : for (int i = MFI.getObjectIndexBegin(); i != 0; ++i)
590 35 : AllocatedFrameSlots.push_back(i);
591 : // Add callee-save objects.
592 592 : for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i)
593 419 : AllocatedFrameSlots.push_back(i);
594 :
595 627 : for (int i : AllocatedFrameSlots) {
596 : // These are converted from int64_t, but they should always fit in int
597 : // because of the FixedCSEnd check above.
598 454 : int ObjOffset = MFI.getObjectOffset(i);
599 454 : int ObjSize = MFI.getObjectSize(i);
600 : int ObjStart, ObjEnd;
601 454 : if (StackGrowsDown) {
602 : // ObjOffset is negative when StackGrowsDown is true.
603 454 : ObjStart = -ObjOffset - ObjSize;
604 : ObjEnd = -ObjOffset;
605 : } else {
606 : ObjStart = ObjOffset;
607 0 : ObjEnd = ObjOffset + ObjSize;
608 : }
609 : // Ignore fixed holes that are in the previous stack frame.
610 454 : if (ObjEnd > 0)
611 426 : StackBytesFree.reset(ObjStart, ObjEnd);
612 : }
613 : }
614 :
615 : /// Assign frame object to an unused portion of the stack in the fixed stack
616 : /// object range. Return true if the allocation was successful.
617 7705870 : static inline bool scavengeStackSlot(MachineFrameInfo &MFI, int FrameIdx,
618 : bool StackGrowsDown, unsigned MaxAlign,
619 : BitVector &StackBytesFree) {
620 7705870 : if (MFI.isVariableSizedObjectIndex(FrameIdx))
621 : return false;
622 :
623 7705374 : if (StackBytesFree.none()) {
624 : // clear it to speed up later scavengeStackSlot calls to
625 : // StackBytesFree.none()
626 : StackBytesFree.clear();
627 7705107 : return false;
628 : }
629 :
630 : unsigned ObjAlign = MFI.getObjectAlignment(FrameIdx);
631 267 : if (ObjAlign > MaxAlign)
632 : return false;
633 :
634 : int64_t ObjSize = MFI.getObjectSize(FrameIdx);
635 : int FreeStart;
636 1813 : for (FreeStart = StackBytesFree.find_first(); FreeStart != -1;
637 1546 : FreeStart = StackBytesFree.find_next(FreeStart)) {
638 :
639 : // Check that free space has suitable alignment.
640 1625 : unsigned ObjStart = StackGrowsDown ? FreeStart + ObjSize : FreeStart;
641 3250 : if (alignTo(ObjStart, ObjAlign) != ObjStart)
642 : continue;
643 :
644 260 : if (FreeStart + ObjSize > StackBytesFree.size())
645 : return false;
646 :
647 : bool AllBytesFree = true;
648 1922 : for (unsigned Byte = 0; Byte < ObjSize; ++Byte)
649 3764 : if (!StackBytesFree.test(FreeStart + Byte)) {
650 : AllBytesFree = false;
651 : break;
652 : }
653 221 : if (AllBytesFree)
654 : break;
655 : }
656 :
657 228 : if (FreeStart == -1)
658 : return false;
659 :
660 40 : if (StackGrowsDown) {
661 40 : int ObjStart = -(FreeStart + ObjSize);
662 : LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP["
663 : << ObjStart << "]\n");
664 40 : MFI.setObjectOffset(FrameIdx, ObjStart);
665 : } else {
666 : LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP["
667 : << FreeStart << "]\n");
668 0 : MFI.setObjectOffset(FrameIdx, FreeStart);
669 : }
670 :
671 40 : StackBytesFree.reset(FreeStart, FreeStart + ObjSize);
672 40 : return true;
673 : }
674 :
675 : /// AssignProtectedObjSet - Helper function to assign large stack objects (i.e.,
676 : /// those required to be close to the Stack Protector) to stack offsets.
677 : static void
678 3750 : AssignProtectedObjSet(const StackObjSet &UnassignedObjs,
679 : SmallSet<int, 16> &ProtectedObjs,
680 : MachineFrameInfo &MFI, bool StackGrowsDown,
681 : int64_t &Offset, unsigned &MaxAlign, unsigned Skew) {
682 :
683 2247 : for (StackObjSet::const_iterator I = UnassignedObjs.begin(),
684 5997 : E = UnassignedObjs.end(); I != E; ++I) {
685 2247 : int i = *I;
686 2247 : AdjustStackOffset(MFI, i, StackGrowsDown, Offset, MaxAlign, Skew);
687 2247 : ProtectedObjs.insert(i);
688 : }
689 3750 : }
690 :
691 : /// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the
692 : /// abstract stack objects.
693 404392 : void PEI::calculateFrameObjectOffsets(MachineFunction &MF) {
694 404392 : const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
695 :
696 : bool StackGrowsDown =
697 404392 : TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
698 :
699 : // Loop over all of the stack objects, assigning sequential addresses...
700 404392 : MachineFrameInfo &MFI = MF.getFrameInfo();
701 :
702 : // Start at the beginning of the local area.
703 : // The Offset is the distance from the stack top in the direction
704 : // of stack growth -- so it's always nonnegative.
705 404392 : int LocalAreaOffset = TFI.getOffsetOfLocalArea();
706 404392 : if (StackGrowsDown)
707 382366 : LocalAreaOffset = -LocalAreaOffset;
708 : assert(LocalAreaOffset >= 0
709 : && "Local area offset should be in direction of stack growth");
710 404392 : int64_t Offset = LocalAreaOffset;
711 :
712 : // Skew to be applied to alignment.
713 404392 : unsigned Skew = TFI.getStackAlignmentSkew(MF);
714 :
715 : // If there are fixed sized objects that are preallocated in the local area,
716 : // non-fixed objects can't be allocated right at the start of local area.
717 : // Adjust 'Offset' to point to the end of last fixed sized preallocated
718 : // object.
719 685230 : for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) {
720 : int64_t FixedOff;
721 280839 : if (StackGrowsDown) {
722 : // The maximum distance from the stack pointer is at lower address of
723 : // the object -- which is given by offset. For down growing stack
724 : // the offset is negative, so we negate the offset to get the distance.
725 279629 : FixedOff = -MFI.getObjectOffset(i);
726 : } else {
727 : // The maximum distance from the start pointer is at the upper
728 : // address of the object.
729 1210 : FixedOff = MFI.getObjectOffset(i) + MFI.getObjectSize(i);
730 : }
731 280839 : if (FixedOff > Offset) Offset = FixedOff;
732 : }
733 :
734 : // First assign frame offsets to stack objects that are used to spill
735 : // callee saved registers.
736 404391 : if (StackGrowsDown) {
737 404829 : for (unsigned i = MinCSFrameIndex; i <= MaxCSFrameIndex; ++i) {
738 : // If the stack grows down, we need to add the size to find the lowest
739 : // address of the object.
740 22463 : Offset += MFI.getObjectSize(i);
741 :
742 : unsigned Align = MFI.getObjectAlignment(i);
743 : // Adjust to alignment boundary
744 22463 : Offset = alignTo(Offset, Align, Skew);
745 :
746 : LLVM_DEBUG(dbgs() << "alloc FI(" << i << ") at SP[" << -Offset << "]\n");
747 22463 : MFI.setObjectOffset(i, -Offset); // Set the computed offset
748 : }
749 22025 : } else if (MaxCSFrameIndex >= MinCSFrameIndex) {
750 : // Be careful about underflow in comparisons agains MinCSFrameIndex.
751 675 : for (unsigned i = MaxCSFrameIndex; i != MinCSFrameIndex - 1; --i) {
752 548 : if (MFI.isDeadObjectIndex(i))
753 : continue;
754 :
755 : unsigned Align = MFI.getObjectAlignment(i);
756 : // Adjust to alignment boundary
757 242 : Offset = alignTo(Offset, Align, Skew);
758 :
759 : LLVM_DEBUG(dbgs() << "alloc FI(" << i << ") at SP[" << Offset << "]\n");
760 : MFI.setObjectOffset(i, Offset);
761 121 : Offset += MFI.getObjectSize(i);
762 : }
763 : }
764 :
765 : // FixedCSEnd is the stack offset to the end of the fixed and callee-save
766 : // stack area.
767 404391 : int64_t FixedCSEnd = Offset;
768 404391 : unsigned MaxAlign = MFI.getMaxAlignment();
769 :
770 : // Make sure the special register scavenging spill slot is closest to the
771 : // incoming stack pointer if a frame pointer is required and is closer
772 : // to the incoming rather than the final stack pointer.
773 404391 : const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
774 609691 : bool EarlyScavengingSlots = (TFI.hasFP(MF) &&
775 410081 : TFI.isFPCloseToIncomingSP() &&
776 813936 : RegInfo->useFPForScavengingIndex(MF) &&
777 204762 : !RegInfo->needsStackRealignment(MF));
778 404392 : if (RS && EarlyScavengingSlots) {
779 : SmallVector<int, 2> SFIs;
780 4498 : RS->getScavengingFrameIndices(SFIs);
781 26 : for (SmallVectorImpl<int>::iterator I = SFIs.begin(),
782 4524 : IE = SFIs.end(); I != IE; ++I)
783 26 : AdjustStackOffset(MFI, *I, StackGrowsDown, Offset, MaxAlign, Skew);
784 : }
785 :
786 : // FIXME: Once this is working, then enable flag will change to a target
787 : // check for whether the frame is large enough to want to use virtual
788 : // frame index registers. Functions which don't want/need this optimization
789 : // will continue to use the existing code path.
790 404392 : if (MFI.getUseLocalStackAllocationBlock()) {
791 13 : unsigned Align = MFI.getLocalFrameMaxAlign();
792 :
793 : // Adjust to alignment boundary.
794 26 : Offset = alignTo(Offset, Align, Skew);
795 :
796 : LLVM_DEBUG(dbgs() << "Local frame base offset: " << Offset << "\n");
797 :
798 : // Resolve offsets for objects in the local block.
799 459 : for (unsigned i = 0, e = MFI.getLocalFrameObjectCount(); i != e; ++i) {
800 446 : std::pair<int, int64_t> Entry = MFI.getLocalFrameObjectMap(i);
801 446 : int64_t FIOffset = (StackGrowsDown ? -Offset : Offset) + Entry.second;
802 : LLVM_DEBUG(dbgs() << "alloc FI(" << Entry.first << ") at SP[" << FIOffset
803 : << "]\n");
804 : MFI.setObjectOffset(Entry.first, FIOffset);
805 : }
806 : // Allocate the local block
807 13 : Offset += MFI.getLocalFrameSize();
808 :
809 13 : MaxAlign = std::max(Align, MaxAlign);
810 : }
811 :
812 : // Retrieve the Exception Handler registration node.
813 : int EHRegNodeFrameIndex = std::numeric_limits<int>::max();
814 404392 : if (const WinEHFuncInfo *FuncInfo = MF.getWinEHFuncInfo())
815 91 : EHRegNodeFrameIndex = FuncInfo->EHRegNodeFrameIndex;
816 :
817 : // Make sure that the stack protector comes before the local variables on the
818 : // stack.
819 404392 : SmallSet<int, 16> ProtectedObjs;
820 404392 : if (MFI.getStackProtectorIndex() >= 0) {
821 : StackObjSet LargeArrayObjs;
822 : StackObjSet SmallArrayObjs;
823 : StackObjSet AddrOfObjs;
824 :
825 1250 : AdjustStackOffset(MFI, MFI.getStackProtectorIndex(), StackGrowsDown,
826 : Offset, MaxAlign, Skew);
827 :
828 : // Assign large stack objects first.
829 6663 : for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
830 10826 : if (MFI.isObjectPreAllocated(i) &&
831 161 : MFI.getUseLocalStackAllocationBlock())
832 : continue;
833 5405 : if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex)
834 : continue;
835 5332 : if (RS && RS->isScavengingFrameIndex((int)i))
836 : continue;
837 5110 : if (MFI.isDeadObjectIndex(i))
838 : continue;
839 4591 : if (MFI.getStackProtectorIndex() == (int)i ||
840 : EHRegNodeFrameIndex == (int)i)
841 : continue;
842 :
843 3345 : switch (MFI.getObjectSSPLayout(i)) {
844 : case MachineFrameInfo::SSPLK_None:
845 : continue;
846 141 : case MachineFrameInfo::SSPLK_SmallArray:
847 141 : SmallArrayObjs.insert(i);
848 141 : continue;
849 1468 : case MachineFrameInfo::SSPLK_AddrOf:
850 1468 : AddrOfObjs.insert(i);
851 1468 : continue;
852 638 : case MachineFrameInfo::SSPLK_LargeArray:
853 638 : LargeArrayObjs.insert(i);
854 638 : continue;
855 : }
856 0 : llvm_unreachable("Unexpected SSPLayoutKind.");
857 : }
858 :
859 1250 : AssignProtectedObjSet(LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown,
860 : Offset, MaxAlign, Skew);
861 1250 : AssignProtectedObjSet(SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown,
862 : Offset, MaxAlign, Skew);
863 1250 : AssignProtectedObjSet(AddrOfObjs, ProtectedObjs, MFI, StackGrowsDown,
864 : Offset, MaxAlign, Skew);
865 : }
866 :
867 : SmallVector<int, 8> ObjectsToAllocate;
868 :
869 : // Then prepare to assign frame offsets to stack objects that are not used to
870 : // spill callee saved registers.
871 8154048 : for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
872 15499312 : if (MFI.isObjectPreAllocated(i) && MFI.getUseLocalStackAllocationBlock())
873 : continue;
874 7749210 : if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex)
875 : continue;
876 7744583 : if (RS && RS->isScavengingFrameIndex((int)i))
877 : continue;
878 7725650 : if (MFI.isDeadObjectIndex(i))
879 : continue;
880 7709397 : if (MFI.getStackProtectorIndex() == (int)i ||
881 : EHRegNodeFrameIndex == (int)i)
882 : continue;
883 7708117 : if (ProtectedObjs.count(i))
884 : continue;
885 :
886 : // Add the objects that we need to allocate to our working set.
887 7705870 : ObjectsToAllocate.push_back(i);
888 : }
889 :
890 : // Allocate the EH registration node first if one is present.
891 404392 : if (EHRegNodeFrameIndex != std::numeric_limits<int>::max())
892 34 : AdjustStackOffset(MFI, EHRegNodeFrameIndex, StackGrowsDown, Offset,
893 : MaxAlign, Skew);
894 :
895 : // Give the targets a chance to order the objects the way they like it.
896 404392 : if (MF.getTarget().getOptLevel() != CodeGenOpt::None &&
897 196875 : MF.getTarget().Options.StackSymbolOrdering)
898 196452 : TFI.orderFrameObjects(MF, ObjectsToAllocate);
899 :
900 : // Keep track of which bytes in the fixed and callee-save range are used so we
901 : // can use the holes when allocating later stack objects. Only do this if
902 : // stack protector isn't being used and the target requests it and we're
903 : // optimizing.
904 : BitVector StackBytesFree;
905 208735 : if (!ObjectsToAllocate.empty() &&
906 208735 : MF.getTarget().getOptLevel() != CodeGenOpt::None &&
907 418156 : MFI.getStackProtectorIndex() < 0 && TFI.enableStackSlotScavenging(MF))
908 173 : computeFreeStackSlots(MFI, StackGrowsDown, MinCSFrameIndex, MaxCSFrameIndex,
909 : FixedCSEnd, StackBytesFree);
910 :
911 : // Now walk the objects and actually assign base offsets to them.
912 8110261 : for (auto &Object : ObjectsToAllocate)
913 7705870 : if (!scavengeStackSlot(MFI, Object, StackGrowsDown, MaxAlign,
914 : StackBytesFree))
915 7705830 : AdjustStackOffset(MFI, Object, StackGrowsDown, Offset, MaxAlign, Skew);
916 :
917 : // Make sure the special register scavenging spill slot is closest to the
918 : // stack pointer.
919 404391 : if (RS && !EarlyScavengingSlots) {
920 : SmallVector<int, 2> SFIs;
921 61834 : RS->getScavengingFrameIndices(SFIs);
922 527 : for (SmallVectorImpl<int>::iterator I = SFIs.begin(),
923 62361 : IE = SFIs.end(); I != IE; ++I)
924 527 : AdjustStackOffset(MFI, *I, StackGrowsDown, Offset, MaxAlign, Skew);
925 : }
926 :
927 404391 : if (!TFI.targetHandlesStackFrameRounding()) {
928 : // If we have reserved argument space for call sites in the function
929 : // immediately on entry to the current function, count it as part of the
930 : // overall stack size.
931 389233 : if (MFI.adjustsStack() && TFI.hasReservedCallFrame(MF))
932 166219 : Offset += MFI.getMaxCallFrameSize();
933 :
934 : // Round up the size to a multiple of the alignment. If the function has
935 : // any calls or alloca's, align to the target's StackAlignment value to
936 : // ensure that the callee's frame or the alloca data is suitably aligned;
937 : // otherwise, for leaf functions, align to the TransientStackAlignment
938 : // value.
939 : unsigned StackAlign;
940 609953 : if (MFI.adjustsStack() || MFI.hasVarSizedObjects() ||
941 221943 : (RegInfo->needsStackRealignment(MF) && MFI.getObjectIndexEnd() != 0))
942 169671 : StackAlign = TFI.getStackAlignment();
943 : else
944 219562 : StackAlign = TFI.getTransientStackAlignment();
945 :
946 : // If the frame pointer is eliminated, all frame offsets will be relative to
947 : // SP not FP. Align to MaxAlign so this works.
948 389233 : StackAlign = std::max(StackAlign, MaxAlign);
949 778466 : Offset = alignTo(Offset, StackAlign, Skew);
950 : }
951 :
952 : // Update frame info to pretend that this is part of the stack...
953 404392 : int64_t StackSize = Offset - LocalAreaOffset;
954 404392 : MFI.setStackSize(StackSize);
955 : NumBytesStackSpace += StackSize;
956 404392 : }
957 :
958 : /// insertPrologEpilogCode - Scan the function for modified callee saved
959 : /// registers, insert spill code for these callee saved registers, then add
960 : /// prolog and epilog code to the function.
961 404310 : void PEI::insertPrologEpilogCode(MachineFunction &MF) {
962 404310 : const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
963 :
964 : // Add prologue to the function...
965 808733 : for (MachineBasicBlock *SaveBlock : SaveBlocks)
966 404428 : TFI.emitPrologue(MF, *SaveBlock);
967 :
968 : // Add epilogue to restore the callee-save registers in each exiting block.
969 807467 : for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
970 403162 : TFI.emitEpilogue(MF, *RestoreBlock);
971 :
972 808728 : for (MachineBasicBlock *SaveBlock : SaveBlocks)
973 404423 : TFI.inlineStackProbe(MF, *SaveBlock);
974 :
975 : // Emit additional code that is required to support segmented stacks, if
976 : // we've been asked for it. This, when linked with a runtime with support
977 : // for segmented stacks (libgcc is one), will result in allocating stack
978 : // space in small chunks instead of one large contiguous block.
979 404305 : if (MF.shouldSplitStack()) {
980 480 : for (MachineBasicBlock *SaveBlock : SaveBlocks)
981 241 : TFI.adjustForSegmentedStacks(MF, *SaveBlock);
982 : // Record that there are split-stack functions, so we will emit a
983 : // special section to tell the linker.
984 239 : MF.getMMI().setHasSplitStack(true);
985 : } else
986 404064 : MF.getMMI().setHasNosplitStack(true);
987 :
988 : // Emit additional code that is required to explicitly handle the stack in
989 : // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The
990 : // approach is rather similar to that of Segmented Stacks, but it uses a
991 : // different conditional check and another BIF for allocating more stack
992 : // space.
993 808606 : if (MF.getFunction().getCallingConv() == CallingConv::HiPE)
994 26 : for (MachineBasicBlock *SaveBlock : SaveBlocks)
995 13 : TFI.adjustForHiPEPrologue(MF, *SaveBlock);
996 404303 : }
997 :
998 : /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical
999 : /// register references and actual offsets.
1000 404385 : void PEI::replaceFrameIndices(MachineFunction &MF) {
1001 404385 : const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1002 404385 : if (!TFI.needsFrameIndexResolution(MF)) return;
1003 :
1004 : // Store SPAdj at exit of a basic block.
1005 : SmallVector<int, 8> SPState;
1006 241647 : SPState.resize(MF.getNumBlockIDs());
1007 : df_iterator_default_set<MachineBasicBlock*> Reachable;
1008 :
1009 : // Iterate over the reachable blocks in DFS order.
1010 241647 : for (auto DFI = df_ext_begin(&MF, Reachable), DFE = df_ext_end(&MF, Reachable);
1011 3334589 : DFI != DFE; ++DFI) {
1012 3092942 : int SPAdj = 0;
1013 : // Check the exit state of the DFS stack predecessor.
1014 3092942 : if (DFI.getPathLength() >= 2) {
1015 2851295 : MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2);
1016 : assert(Reachable.count(StackPred) &&
1017 : "DFS stack predecessor is already visited.\n");
1018 5702590 : SPAdj = SPState[StackPred->getNumber()];
1019 : }
1020 3092942 : MachineBasicBlock *BB = *DFI;
1021 3092942 : replaceFrameIndices(BB, MF, SPAdj);
1022 6185884 : SPState[BB->getNumber()] = SPAdj;
1023 : }
1024 :
1025 : // Handle the unreachable blocks.
1026 3334660 : for (auto &BB : MF) {
1027 3093013 : if (Reachable.count(&BB))
1028 : // Already handled in DFS traversal.
1029 3092942 : continue;
1030 71 : int SPAdj = 0;
1031 71 : replaceFrameIndices(&BB, MF, SPAdj);
1032 : }
1033 : }
1034 :
1035 0 : void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
1036 : int &SPAdj) {
1037 : assert(MF.getSubtarget().getRegisterInfo() &&
1038 : "getRegisterInfo() must be implemented!");
1039 0 : const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1040 0 : const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1041 0 : const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
1042 :
1043 0 : if (RS && FrameIndexEliminationScavenging)
1044 0 : RS->enterBasicBlock(*BB);
1045 :
1046 : bool InsideCallSequence = false;
1047 :
1048 0 : for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) {
1049 0 : if (TII.isFrameInstr(*I)) {
1050 : InsideCallSequence = TII.isFrameSetup(*I);
1051 0 : SPAdj += TII.getSPAdjust(*I);
1052 0 : I = TFI->eliminateCallFramePseudoInstr(MF, *BB, I);
1053 0 : continue;
1054 : }
1055 :
1056 : MachineInstr &MI = *I;
1057 : bool DoIncr = true;
1058 : bool DidFinishLoop = true;
1059 0 : for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
1060 0 : if (!MI.getOperand(i).isFI())
1061 0 : continue;
1062 :
1063 : // Frame indices in debug values are encoded in a target independent
1064 : // way with simply the frame index and offset rather than any
1065 : // target-specific addressing mode.
1066 0 : if (MI.isDebugValue()) {
1067 : assert(i == 0 && "Frame indices can only appear as the first "
1068 : "operand of a DBG_VALUE machine instruction");
1069 : unsigned Reg;
1070 : int64_t Offset =
1071 0 : TFI->getFrameIndexReference(MF, MI.getOperand(0).getIndex(), Reg);
1072 0 : MI.getOperand(0).ChangeToRegister(Reg, false /*isDef*/);
1073 0 : MI.getOperand(0).setIsDebug();
1074 0 : auto *DIExpr = DIExpression::prepend(MI.getDebugExpression(),
1075 : DIExpression::NoDeref, Offset);
1076 0 : MI.getOperand(3).setMetadata(DIExpr);
1077 0 : continue;
1078 : }
1079 :
1080 : // TODO: This code should be commoned with the code for
1081 : // PATCHPOINT. There's no good reason for the difference in
1082 : // implementation other than historical accident. The only
1083 : // remaining difference is the unconditional use of the stack
1084 : // pointer as the base register.
1085 0 : if (MI.getOpcode() == TargetOpcode::STATEPOINT) {
1086 : assert((!MI.isDebugValue() || i == 0) &&
1087 : "Frame indicies can only appear as the first operand of a "
1088 : "DBG_VALUE machine instruction");
1089 : unsigned Reg;
1090 0 : MachineOperand &Offset = MI.getOperand(i + 1);
1091 0 : int refOffset = TFI->getFrameIndexReferencePreferSP(
1092 0 : MF, MI.getOperand(i).getIndex(), Reg, /*IgnoreSPUpdates*/ false);
1093 0 : Offset.setImm(Offset.getImm() + refOffset);
1094 0 : MI.getOperand(i).ChangeToRegister(Reg, false /*isDef*/);
1095 0 : continue;
1096 : }
1097 :
1098 : // Some instructions (e.g. inline asm instructions) can have
1099 : // multiple frame indices and/or cause eliminateFrameIndex
1100 : // to insert more than one instruction. We need the register
1101 : // scavenger to go through all of these instructions so that
1102 : // it can update its register information. We keep the
1103 : // iterator at the point before insertion so that we can
1104 : // revisit them in full.
1105 : bool AtBeginning = (I == BB->begin());
1106 0 : if (!AtBeginning) --I;
1107 :
1108 : // If this instruction has a FrameIndex operand, we need to
1109 : // use that target machine register info object to eliminate
1110 : // it.
1111 0 : TRI.eliminateFrameIndex(MI, SPAdj, i,
1112 0 : FrameIndexEliminationScavenging ? RS : nullptr);
1113 :
1114 : // Reset the iterator if we were at the beginning of the BB.
1115 0 : if (AtBeginning) {
1116 0 : I = BB->begin();
1117 : DoIncr = false;
1118 : }
1119 :
1120 : DidFinishLoop = false;
1121 : break;
1122 : }
1123 :
1124 : // If we are looking at a call sequence, we need to keep track of
1125 : // the SP adjustment made by each instruction in the sequence.
1126 : // This includes both the frame setup/destroy pseudos (handled above),
1127 : // as well as other instructions that have side effects w.r.t the SP.
1128 : // Note that this must come after eliminateFrameIndex, because
1129 : // if I itself referred to a frame index, we shouldn't count its own
1130 : // adjustment.
1131 0 : if (DidFinishLoop && InsideCallSequence)
1132 0 : SPAdj += TII.getSPAdjust(MI);
1133 :
1134 0 : if (DoIncr && I != BB->end()) ++I;
1135 :
1136 : // Update register states.
1137 0 : if (RS && FrameIndexEliminationScavenging && DidFinishLoop)
1138 0 : RS->forward(MI);
1139 : }
1140 0 : }
|
