LLVM 18.0.0git
X86FrameLowering.cpp
Go to the documentation of this file.
1//===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86FrameLowering.h"
15#include "X86InstrBuilder.h"
16#include "X86InstrInfo.h"
18#include "X86Subtarget.h"
19#include "X86TargetMachine.h"
20#include "llvm/ADT/Statistic.h"
28#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/Function.h"
31#include "llvm/MC/MCAsmInfo.h"
33#include "llvm/MC/MCSymbol.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/LEB128.h"
37#include <cstdlib>
38
39#define DEBUG_TYPE "x86-fl"
40
41STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue");
42STATISTIC(NumFrameExtraProbe,
43 "Number of extra stack probes generated in prologue");
44STATISTIC(NumFunctionUsingPush2Pop2, "Number of funtions using push2/pop2");
45
46using namespace llvm;
47
49 MaybeAlign StackAlignOverride)
50 : TargetFrameLowering(StackGrowsDown, StackAlignOverride.valueOrOne(),
51 STI.is64Bit() ? -8 : -4),
52 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) {
53 // Cache a bunch of frame-related predicates for this subtarget.
55 Is64Bit = STI.is64Bit();
57 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
60}
61
63 return !MF.getFrameInfo().hasVarSizedObjects() &&
64 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences() &&
65 !MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall();
66}
67
68/// canSimplifyCallFramePseudos - If there is a reserved call frame, the
69/// call frame pseudos can be simplified. Having a FP, as in the default
70/// implementation, is not sufficient here since we can't always use it.
71/// Use a more nuanced condition.
73 const MachineFunction &MF) const {
74 return hasReservedCallFrame(MF) ||
75 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
76 (hasFP(MF) && !TRI->hasStackRealignment(MF)) ||
78}
79
80// needsFrameIndexResolution - Do we need to perform FI resolution for
81// this function. Normally, this is required only when the function
82// has any stack objects. However, FI resolution actually has another job,
83// not apparent from the title - it resolves callframesetup/destroy
84// that were not simplified earlier.
85// So, this is required for x86 functions that have push sequences even
86// when there are no stack objects.
88 const MachineFunction &MF) const {
89 return MF.getFrameInfo().hasStackObjects() ||
90 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
91}
92
93/// hasFP - Return true if the specified function should have a dedicated frame
94/// pointer register. This is true if the function has variable sized allocas
95/// or if frame pointer elimination is disabled.
97 const MachineFrameInfo &MFI = MF.getFrameInfo();
98 return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
99 TRI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
103 MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() ||
104 MFI.hasStackMap() || MFI.hasPatchPoint() ||
105 (isWin64Prologue(MF) && MFI.hasCopyImplyingStackAdjustment()));
106}
107
108static unsigned getSUBriOpcode(bool IsLP64) {
109 return IsLP64 ? X86::SUB64ri32 : X86::SUB32ri;
110}
111
112static unsigned getADDriOpcode(bool IsLP64) {
113 return IsLP64 ? X86::ADD64ri32 : X86::ADD32ri;
114}
115
116static unsigned getSUBrrOpcode(bool IsLP64) {
117 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
118}
119
120static unsigned getADDrrOpcode(bool IsLP64) {
121 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
122}
123
124static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) {
125 return IsLP64 ? X86::AND64ri32 : X86::AND32ri;
126}
127
128static unsigned getLEArOpcode(bool IsLP64) {
129 return IsLP64 ? X86::LEA64r : X86::LEA32r;
130}
131
132static unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm) {
133 if (Use64BitReg) {
134 if (isUInt<32>(Imm))
135 return X86::MOV32ri64;
136 if (isInt<32>(Imm))
137 return X86::MOV64ri32;
138 return X86::MOV64ri;
139 }
140 return X86::MOV32ri;
141}
142
143// Push-Pop Acceleration (PPX) hint is used to indicate that the POP reads the
144// value written by the PUSH from the stack. The processor tracks these marked
145// instructions internally and fast-forwards register data between matching PUSH
146// and POP instructions, without going through memory or through the training
147// loop of the Fast Store Forwarding Predictor (FSFP). Instead, a more efficient
148// memory-renaming optimization can be used.
149//
150// The PPX hint is purely a performance hint. Instructions with this hint have
151// the same functional semantics as those without. PPX hints set by the
152// compiler that violate the balancing rule may turn off the PPX optimization,
153// but they will not affect program semantics.
154//
155// Hence, PPX is used for balanced spill/reloads (Exceptions and setjmp/longjmp
156// are not considered).
157//
158// PUSH2 and POP2 are instructions for (respectively) pushing/popping 2
159// GPRs at a time to/from the stack.
160static unsigned getPUSHOpcode(const X86Subtarget &ST) {
161 return ST.is64Bit() ? (ST.hasPPX() ? X86::PUSHP64r : X86::PUSH64r)
162 : X86::PUSH32r;
163}
164static unsigned getPOPOpcode(const X86Subtarget &ST) {
165 return ST.is64Bit() ? (ST.hasPPX() ? X86::POPP64r : X86::POP64r)
166 : X86::POP32r;
167}
168static unsigned getPUSH2Opcode(const X86Subtarget &ST) {
169 return ST.hasPPX() ? X86::PUSH2P : X86::PUSH2;
170}
171static unsigned getPOP2Opcode(const X86Subtarget &ST) {
172 return ST.hasPPX() ? X86::POP2P : X86::POP2;
173}
174
177 unsigned Reg = RegMask.PhysReg;
178
179 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
180 Reg == X86::AH || Reg == X86::AL)
181 return true;
182 }
183
184 return false;
185}
186
187/// Check if the flags need to be preserved before the terminators.
188/// This would be the case, if the eflags is live-in of the region
189/// composed by the terminators or live-out of that region, without
190/// being defined by a terminator.
191static bool
193 for (const MachineInstr &MI : MBB.terminators()) {
194 bool BreakNext = false;
195 for (const MachineOperand &MO : MI.operands()) {
196 if (!MO.isReg())
197 continue;
198 Register Reg = MO.getReg();
199 if (Reg != X86::EFLAGS)
200 continue;
201
202 // This terminator needs an eflags that is not defined
203 // by a previous another terminator:
204 // EFLAGS is live-in of the region composed by the terminators.
205 if (!MO.isDef())
206 return true;
207 // This terminator defines the eflags, i.e., we don't need to preserve it.
208 // However, we still need to check this specific terminator does not
209 // read a live-in value.
210 BreakNext = true;
211 }
212 // We found a definition of the eflags, no need to preserve them.
213 if (BreakNext)
214 return false;
215 }
216
217 // None of the terminators use or define the eflags.
218 // Check if they are live-out, that would imply we need to preserve them.
219 for (const MachineBasicBlock *Succ : MBB.successors())
220 if (Succ->isLiveIn(X86::EFLAGS))
221 return true;
222
223 return false;
224}
225
226/// emitSPUpdate - Emit a series of instructions to increment / decrement the
227/// stack pointer by a constant value.
230 const DebugLoc &DL, int64_t NumBytes,
231 bool InEpilogue) const {
232 bool isSub = NumBytes < 0;
233 uint64_t Offset = isSub ? -NumBytes : NumBytes;
236
237 uint64_t Chunk = (1LL << 31) - 1;
238
242 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
243
244 // It's ok to not take into account large chunks when probing, as the
245 // allocation is split in smaller chunks anyway.
246 if (EmitInlineStackProbe && !InEpilogue) {
247
248 // This pseudo-instruction is going to be expanded, potentially using a
249 // loop, by inlineStackProbe().
250 BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING)).addImm(Offset);
251 return;
252 } else if (Offset > Chunk) {
253 // Rather than emit a long series of instructions for large offsets,
254 // load the offset into a register and do one sub/add
255 unsigned Reg = 0;
256 unsigned Rax = (unsigned)(Is64Bit ? X86::RAX : X86::EAX);
257
258 if (isSub && !isEAXLiveIn(MBB))
259 Reg = Rax;
260 else
262
263 unsigned AddSubRROpc =
265 if (Reg) {
267 .addImm(Offset)
268 .setMIFlag(Flag);
269 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AddSubRROpc), StackPtr)
271 .addReg(Reg);
272 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
273 return;
274 } else if (Offset > 8 * Chunk) {
275 // If we would need more than 8 add or sub instructions (a >16GB stack
276 // frame), it's worth spilling RAX to materialize this immediate.
277 // pushq %rax
278 // movabsq +-$Offset+-SlotSize, %rax
279 // addq %rsp, %rax
280 // xchg %rax, (%rsp)
281 // movq (%rsp), %rsp
282 assert(Is64Bit && "can't have 32-bit 16GB stack frame");
283 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
285 .setMIFlag(Flag);
286 // Subtract is not commutative, so negate the offset and always use add.
287 // Subtract 8 less and add 8 more to account for the PUSH we just did.
288 if (isSub)
289 Offset = -(Offset - SlotSize);
290 else
293 .addImm(Offset)
294 .setMIFlag(Flag);
295 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(X86::ADD64rr), Rax)
296 .addReg(Rax)
298 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
299 // Exchange the new SP in RAX with the top of the stack.
301 BuildMI(MBB, MBBI, DL, TII.get(X86::XCHG64rm), Rax).addReg(Rax),
302 StackPtr, false, 0);
303 // Load new SP from the top of the stack into RSP.
304 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), StackPtr),
305 StackPtr, false, 0);
306 return;
307 }
308 }
309
310 while (Offset) {
311 uint64_t ThisVal = std::min(Offset, Chunk);
312 if (ThisVal == SlotSize) {
313 // Use push / pop for slot sized adjustments as a size optimization. We
314 // need to find a dead register when using pop.
315 unsigned Reg = isSub ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
317 if (Reg) {
318 unsigned Opc = isSub ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
319 : (Is64Bit ? X86::POP64r : X86::POP32r);
320 BuildMI(MBB, MBBI, DL, TII.get(Opc))
321 .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub))
322 .setMIFlag(Flag);
323 Offset -= ThisVal;
324 continue;
325 }
326 }
327
328 BuildStackAdjustment(MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue)
329 .setMIFlag(Flag);
330
331 Offset -= ThisVal;
332 }
333}
334
335MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
337 const DebugLoc &DL, int64_t Offset, bool InEpilogue) const {
338 assert(Offset != 0 && "zero offset stack adjustment requested");
339
340 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
341 // is tricky.
342 bool UseLEA;
343 if (!InEpilogue) {
344 // Check if inserting the prologue at the beginning
345 // of MBB would require to use LEA operations.
346 // We need to use LEA operations if EFLAGS is live in, because
347 // it means an instruction will read it before it gets defined.
348 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(X86::EFLAGS);
349 } else {
350 // If we can use LEA for SP but we shouldn't, check that none
351 // of the terminators uses the eflags. Otherwise we will insert
352 // a ADD that will redefine the eflags and break the condition.
353 // Alternatively, we could move the ADD, but this may not be possible
354 // and is an optimization anyway.
356 if (UseLEA && !STI.useLeaForSP())
358 // If that assert breaks, that means we do not do the right thing
359 // in canUseAsEpilogue.
361 "We shouldn't have allowed this insertion point");
362 }
363
365 if (UseLEA) {
368 StackPtr),
369 StackPtr, false, Offset);
370 } else {
371 bool IsSub = Offset < 0;
372 uint64_t AbsOffset = IsSub ? -Offset : Offset;
373 const unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr)
375 MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
377 .addImm(AbsOffset);
378 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
379 }
380 return MI;
381}
382
385 bool doMergeWithPrevious) const {
386 if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
387 (!doMergeWithPrevious && MBBI == MBB.end()))
388 return 0;
389
390 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI;
391
393 // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI
394 // instruction, and that there are no DBG_VALUE or other instructions between
395 // ADD/SUB/LEA and its corresponding CFI instruction.
396 /* TODO: Add support for the case where there are multiple CFI instructions
397 below the ADD/SUB/LEA, e.g.:
398 ...
399 add
400 cfi_def_cfa_offset
401 cfi_offset
402 ...
403 */
404 if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction())
405 PI = std::prev(PI);
406
407 unsigned Opc = PI->getOpcode();
408 int Offset = 0;
409
410 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri) &&
411 PI->getOperand(0).getReg() == StackPtr) {
412 assert(PI->getOperand(1).getReg() == StackPtr);
413 Offset = PI->getOperand(2).getImm();
414 } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
415 PI->getOperand(0).getReg() == StackPtr &&
416 PI->getOperand(1).getReg() == StackPtr &&
417 PI->getOperand(2).getImm() == 1 &&
418 PI->getOperand(3).getReg() == X86::NoRegister &&
419 PI->getOperand(5).getReg() == X86::NoRegister) {
420 // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
421 Offset = PI->getOperand(4).getImm();
422 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri) &&
423 PI->getOperand(0).getReg() == StackPtr) {
424 assert(PI->getOperand(1).getReg() == StackPtr);
425 Offset = -PI->getOperand(2).getImm();
426 } else
427 return 0;
428
429 PI = MBB.erase(PI);
430 if (PI != MBB.end() && PI->isCFIInstruction()) {
431 auto CIs = MBB.getParent()->getFrameInstructions();
432 MCCFIInstruction CI = CIs[PI->getOperand(0).getCFIIndex()];
435 PI = MBB.erase(PI);
436 }
437 if (!doMergeWithPrevious)
439
440 return Offset;
441}
442
445 const DebugLoc &DL,
446 const MCCFIInstruction &CFIInst,
447 MachineInstr::MIFlag Flag) const {
449 unsigned CFIIndex = MF.addFrameInst(CFIInst);
450
452 MF.getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
453
454 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
455 .addCFIIndex(CFIIndex)
456 .setMIFlag(Flag);
457}
458
459/// Emits Dwarf Info specifying offsets of callee saved registers and
460/// frame pointer. This is called only when basic block sections are enabled.
464 if (!hasFP(MF)) {
466 return;
467 }
468 const MachineModuleInfo &MMI = MF.getMMI();
471 const Register MachineFramePtr =
473 : FramePtr;
474 unsigned DwarfReg = MRI->getDwarfRegNum(MachineFramePtr, true);
475 // Offset = space for return address + size of the frame pointer itself.
476 unsigned Offset = (Is64Bit ? 8 : 4) + (Uses64BitFramePtr ? 8 : 4);
478 MCCFIInstruction::createOffset(nullptr, DwarfReg, -Offset));
480}
481
484 const DebugLoc &DL, bool IsPrologue) const {
486 MachineFrameInfo &MFI = MF.getFrameInfo();
487 MachineModuleInfo &MMI = MF.getMMI();
490
491 // Add callee saved registers to move list.
492 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
493
494 // Calculate offsets.
495 for (const CalleeSavedInfo &I : CSI) {
496 int64_t Offset = MFI.getObjectOffset(I.getFrameIdx());
497 Register Reg = I.getReg();
498 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
499
500 if (IsPrologue) {
501 if (X86FI->getStackPtrSaveMI()) {
502 // +2*SlotSize because there is return address and ebp at the bottom
503 // of the stack.
504 // | retaddr |
505 // | ebp |
506 // | |<--ebp
507 Offset += 2 * SlotSize;
508 SmallString<64> CfaExpr;
509 CfaExpr.push_back(dwarf::DW_CFA_expression);
510 uint8_t buffer[16];
511 CfaExpr.append(buffer, buffer + encodeULEB128(DwarfReg, buffer));
512 CfaExpr.push_back(2);
514 const Register MachineFramePtr =
517 : FramePtr;
518 unsigned DwarfFramePtr = MRI->getDwarfRegNum(MachineFramePtr, true);
519 CfaExpr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
520 CfaExpr.append(buffer, buffer + encodeSLEB128(Offset, buffer));
522 MCCFIInstruction::createEscape(nullptr, CfaExpr.str()),
524 } else {
526 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
527 }
528 } else {
530 MCCFIInstruction::createRestore(nullptr, DwarfReg));
531 }
532 }
533 if (auto *MI = X86FI->getStackPtrSaveMI()) {
534 int FI = MI->getOperand(1).getIndex();
535 int64_t Offset = MFI.getObjectOffset(FI) + 2 * SlotSize;
536 SmallString<64> CfaExpr;
538 const Register MachineFramePtr =
541 : FramePtr;
542 unsigned DwarfFramePtr = MRI->getDwarfRegNum(MachineFramePtr, true);
543 CfaExpr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
544 uint8_t buffer[16];
545 CfaExpr.append(buffer, buffer + encodeSLEB128(Offset, buffer));
546 CfaExpr.push_back(dwarf::DW_OP_deref);
547
548 SmallString<64> DefCfaExpr;
549 DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
550 DefCfaExpr.append(buffer, buffer + encodeSLEB128(CfaExpr.size(), buffer));
551 DefCfaExpr.append(CfaExpr.str());
552 // DW_CFA_def_cfa_expression: DW_OP_breg5 offset, DW_OP_deref
554 MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str()),
556 }
557}
558
559void X86FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
560 MachineBasicBlock &MBB) const {
561 const MachineFunction &MF = *MBB.getParent();
562
563 // Insertion point.
565
566 // Fake a debug loc.
567 DebugLoc DL;
568 if (MBBI != MBB.end())
569 DL = MBBI->getDebugLoc();
570
571 // Zero out FP stack if referenced. Do this outside of the loop below so that
572 // it's done only once.
573 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
574 for (MCRegister Reg : RegsToZero.set_bits()) {
575 if (!X86::RFP80RegClass.contains(Reg))
576 continue;
577
578 unsigned NumFPRegs = ST.is64Bit() ? 8 : 7;
579 for (unsigned i = 0; i != NumFPRegs; ++i)
580 BuildMI(MBB, MBBI, DL, TII.get(X86::LD_F0));
581
582 for (unsigned i = 0; i != NumFPRegs; ++i)
583 BuildMI(MBB, MBBI, DL, TII.get(X86::ST_FPrr)).addReg(X86::ST0);
584 break;
585 }
586
587 // For GPRs, we only care to clear out the 32-bit register.
588 BitVector GPRsToZero(TRI->getNumRegs());
589 for (MCRegister Reg : RegsToZero.set_bits())
590 if (TRI->isGeneralPurposeRegister(MF, Reg)) {
591 GPRsToZero.set(getX86SubSuperRegister(Reg, 32));
592 RegsToZero.reset(Reg);
593 }
594
595 // Zero out the GPRs first.
596 for (MCRegister Reg : GPRsToZero.set_bits())
598
599 // Zero out the remaining registers.
600 for (MCRegister Reg : RegsToZero.set_bits())
602}
603
606 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
607 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
610 if (InProlog) {
611 BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING))
612 .addImm(0 /* no explicit stack size */);
613 } else {
614 emitStackProbeInline(MF, MBB, MBBI, DL, false);
615 }
616 } else {
617 emitStackProbeCall(MF, MBB, MBBI, DL, InProlog, InstrNum);
618 }
619}
620
622 return STI.isOSWindows() && !STI.isTargetWin64();
623}
624
626 MachineBasicBlock &PrologMBB) const {
627 auto Where = llvm::find_if(PrologMBB, [](MachineInstr &MI) {
628 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
629 });
630 if (Where != PrologMBB.end()) {
631 DebugLoc DL = PrologMBB.findDebugLoc(Where);
632 emitStackProbeInline(MF, PrologMBB, Where, DL, true);
633 Where->eraseFromParent();
634 }
635}
636
637void X86FrameLowering::emitStackProbeInline(MachineFunction &MF,
640 const DebugLoc &DL,
641 bool InProlog) const {
643 if (STI.isTargetWindowsCoreCLR() && STI.is64Bit())
644 emitStackProbeInlineWindowsCoreCLR64(MF, MBB, MBBI, DL, InProlog);
645 else
646 emitStackProbeInlineGeneric(MF, MBB, MBBI, DL, InProlog);
647}
648
649void X86FrameLowering::emitStackProbeInlineGeneric(
651 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
652 MachineInstr &AllocWithProbe = *MBBI;
653 uint64_t Offset = AllocWithProbe.getOperand(0).getImm();
654
657 assert(!(STI.is64Bit() && STI.isTargetWindowsCoreCLR()) &&
658 "different expansion expected for CoreCLR 64 bit");
659
660 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
661 uint64_t ProbeChunk = StackProbeSize * 8;
662
663 uint64_t MaxAlign =
664 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
665
666 // Synthesize a loop or unroll it, depending on the number of iterations.
667 // BuildStackAlignAND ensures that only MaxAlign % StackProbeSize bits left
668 // between the unaligned rsp and current rsp.
669 if (Offset > ProbeChunk) {
670 emitStackProbeInlineGenericLoop(MF, MBB, MBBI, DL, Offset,
671 MaxAlign % StackProbeSize);
672 } else {
673 emitStackProbeInlineGenericBlock(MF, MBB, MBBI, DL, Offset,
674 MaxAlign % StackProbeSize);
675 }
676}
677
678void X86FrameLowering::emitStackProbeInlineGenericBlock(
681 uint64_t AlignOffset) const {
682
683 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
684 const bool HasFP = hasFP(MF);
687 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
688 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
689
690 uint64_t CurrentOffset = 0;
691
692 assert(AlignOffset < StackProbeSize);
693
694 // If the offset is so small it fits within a page, there's nothing to do.
695 if (StackProbeSize < Offset + AlignOffset) {
696
697 uint64_t StackAdjustment = StackProbeSize - AlignOffset;
698 BuildStackAdjustment(MBB, MBBI, DL, -StackAdjustment, /*InEpilogue=*/false)
700 if (!HasFP && NeedsDwarfCFI) {
701 BuildCFI(
702 MBB, MBBI, DL,
703 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment));
704 }
705
706 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
708 StackPtr, false, 0)
709 .addImm(0)
711 NumFrameExtraProbe++;
712 CurrentOffset = StackProbeSize - AlignOffset;
713 }
714
715 // For the next N - 1 pages, just probe. I tried to take advantage of
716 // natural probes but it implies much more logic and there was very few
717 // interesting natural probes to interleave.
718 while (CurrentOffset + StackProbeSize < Offset) {
719 BuildStackAdjustment(MBB, MBBI, DL, -StackProbeSize, /*InEpilogue=*/false)
721
722 if (!HasFP && NeedsDwarfCFI) {
723 BuildCFI(
724 MBB, MBBI, DL,
725 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackProbeSize));
726 }
727 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
729 StackPtr, false, 0)
730 .addImm(0)
732 NumFrameExtraProbe++;
733 CurrentOffset += StackProbeSize;
734 }
735
736 // No need to probe the tail, it is smaller than a Page.
737 uint64_t ChunkSize = Offset - CurrentOffset;
738 if (ChunkSize == SlotSize) {
739 // Use push for slot sized adjustments as a size optimization,
740 // like emitSPUpdate does when not probing.
741 unsigned Reg = Is64Bit ? X86::RAX : X86::EAX;
742 unsigned Opc = Is64Bit ? X86::PUSH64r : X86::PUSH32r;
743 BuildMI(MBB, MBBI, DL, TII.get(Opc))
746 } else {
747 BuildStackAdjustment(MBB, MBBI, DL, -ChunkSize, /*InEpilogue=*/false)
749 }
750 // No need to adjust Dwarf CFA offset here, the last position of the stack has
751 // been defined
752}
753
754void X86FrameLowering::emitStackProbeInlineGenericLoop(
757 uint64_t AlignOffset) const {
758 assert(Offset && "null offset");
759
760 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
762 "Inline stack probe loop will clobber live EFLAGS.");
763
764 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
765 const bool HasFP = hasFP(MF);
768 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
769 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
770
771 if (AlignOffset) {
772 if (AlignOffset < StackProbeSize) {
773 // Perform a first smaller allocation followed by a probe.
774 BuildStackAdjustment(MBB, MBBI, DL, -AlignOffset, /*InEpilogue=*/false)
776
777 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
779 StackPtr, false, 0)
780 .addImm(0)
782 NumFrameExtraProbe++;
783 Offset -= AlignOffset;
784 }
785 }
786
787 // Synthesize a loop
788 NumFrameLoopProbe++;
789 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
790
791 MachineBasicBlock *testMBB = MF.CreateMachineBasicBlock(LLVM_BB);
792 MachineBasicBlock *tailMBB = MF.CreateMachineBasicBlock(LLVM_BB);
793
795 MF.insert(MBBIter, testMBB);
796 MF.insert(MBBIter, tailMBB);
797
798 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
799 : Is64Bit ? X86::R11D
800 : X86::EAX;
801
802 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::COPY), FinalStackProbed)
805
806 // save loop bound
807 {
808 const unsigned BoundOffset = alignDown(Offset, StackProbeSize);
809 const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr);
810 BuildMI(MBB, MBBI, DL, TII.get(SUBOpc), FinalStackProbed)
811 .addReg(FinalStackProbed)
812 .addImm(BoundOffset)
814
815 // while in the loop, use loop-invariant reg for CFI,
816 // instead of the stack pointer, which changes during the loop
817 if (!HasFP && NeedsDwarfCFI) {
818 // x32 uses the same DWARF register numbers as x86-64,
819 // so there isn't a register number for r11d, we must use r11 instead
820 const Register DwarfFinalStackProbed =
822 ? Register(getX86SubSuperRegister(FinalStackProbed, 64))
823 : FinalStackProbed;
824
827 nullptr, TRI->getDwarfRegNum(DwarfFinalStackProbed, true)));
829 MCCFIInstruction::createAdjustCfaOffset(nullptr, BoundOffset));
830 }
831 }
832
833 // allocate a page
834 BuildStackAdjustment(*testMBB, testMBB->end(), DL, -StackProbeSize,
835 /*InEpilogue=*/false)
837
838 // touch the page
839 addRegOffset(BuildMI(testMBB, DL, TII.get(MovMIOpc))
841 StackPtr, false, 0)
842 .addImm(0)
844
845 // cmp with stack pointer bound
846 BuildMI(testMBB, DL, TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
848 .addReg(FinalStackProbed)
850
851 // jump
852 BuildMI(testMBB, DL, TII.get(X86::JCC_1))
853 .addMBB(testMBB)
856 testMBB->addSuccessor(testMBB);
857 testMBB->addSuccessor(tailMBB);
858
859 // BB management
860 tailMBB->splice(tailMBB->end(), &MBB, MBBI, MBB.end());
862 MBB.addSuccessor(testMBB);
863
864 // handle tail
865 const uint64_t TailOffset = Offset % StackProbeSize;
866 MachineBasicBlock::iterator TailMBBIter = tailMBB->begin();
867 if (TailOffset) {
868 BuildStackAdjustment(*tailMBB, TailMBBIter, DL, -TailOffset,
869 /*InEpilogue=*/false)
871 }
872
873 // after the loop, switch back to stack pointer for CFI
874 if (!HasFP && NeedsDwarfCFI) {
875 // x32 uses the same DWARF register numbers as x86-64,
876 // so there isn't a register number for esp, we must use rsp instead
877 const Register DwarfStackPtr =
881
882 BuildCFI(*tailMBB, TailMBBIter, DL,
884 nullptr, TRI->getDwarfRegNum(DwarfStackPtr, true)));
885 }
886
887 // Update Live In information
888 recomputeLiveIns(*testMBB);
889 recomputeLiveIns(*tailMBB);
890}
891
892void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
894 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
896 assert(STI.is64Bit() && "different expansion needed for 32 bit");
897 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR");
899 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
900
901 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
903 "Inline stack probe loop will clobber live EFLAGS.");
904
905 // RAX contains the number of bytes of desired stack adjustment.
906 // The handling here assumes this value has already been updated so as to
907 // maintain stack alignment.
908 //
909 // We need to exit with RSP modified by this amount and execute suitable
910 // page touches to notify the OS that we're growing the stack responsibly.
911 // All stack probing must be done without modifying RSP.
912 //
913 // MBB:
914 // SizeReg = RAX;
915 // ZeroReg = 0
916 // CopyReg = RSP
917 // Flags, TestReg = CopyReg - SizeReg
918 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg
919 // LimitReg = gs magic thread env access
920 // if FinalReg >= LimitReg goto ContinueMBB
921 // RoundBB:
922 // RoundReg = page address of FinalReg
923 // LoopMBB:
924 // LoopReg = PHI(LimitReg,ProbeReg)
925 // ProbeReg = LoopReg - PageSize
926 // [ProbeReg] = 0
927 // if (ProbeReg > RoundReg) goto LoopMBB
928 // ContinueMBB:
929 // RSP = RSP - RAX
930 // [rest of original MBB]
931
932 // Set up the new basic blocks
933 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(LLVM_BB);
934 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
935 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB);
936
937 MachineFunction::iterator MBBIter = std::next(MBB.getIterator());
938 MF.insert(MBBIter, RoundMBB);
939 MF.insert(MBBIter, LoopMBB);
940 MF.insert(MBBIter, ContinueMBB);
941
942 // Split MBB and move the tail portion down to ContinueMBB.
943 MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI);
944 ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end());
946
947 // Some useful constants
948 const int64_t ThreadEnvironmentStackLimit = 0x10;
949 const int64_t PageSize = 0x1000;
950 const int64_t PageMask = ~(PageSize - 1);
951
952 // Registers we need. For the normal case we use virtual
953 // registers. For the prolog expansion we use RAX, RCX and RDX.
955 const TargetRegisterClass *RegClass = &X86::GR64RegClass;
956 const Register
957 SizeReg = InProlog ? X86::RAX : MRI.createVirtualRegister(RegClass),
958 ZeroReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
959 CopyReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
960 TestReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
961 FinalReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
962 RoundedReg = InProlog ? X86::RDX : MRI.createVirtualRegister(RegClass),
963 LimitReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
964 JoinReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass),
965 ProbeReg = InProlog ? X86::RCX : MRI.createVirtualRegister(RegClass);
966
967 // SP-relative offsets where we can save RCX and RDX.
968 int64_t RCXShadowSlot = 0;
969 int64_t RDXShadowSlot = 0;
970
971 // If inlining in the prolog, save RCX and RDX.
972 if (InProlog) {
973 // Compute the offsets. We need to account for things already
974 // pushed onto the stack at this point: return address, frame
975 // pointer (if used), and callee saves.
977 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize();
978 const bool HasFP = hasFP(MF);
979
980 // Check if we need to spill RCX and/or RDX.
981 // Here we assume that no earlier prologue instruction changes RCX and/or
982 // RDX, so checking the block live-ins is enough.
983 const bool IsRCXLiveIn = MBB.isLiveIn(X86::RCX);
984 const bool IsRDXLiveIn = MBB.isLiveIn(X86::RDX);
985 int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0);
986 // Assign the initial slot to both registers, then change RDX's slot if both
987 // need to be spilled.
988 if (IsRCXLiveIn)
989 RCXShadowSlot = InitSlot;
990 if (IsRDXLiveIn)
991 RDXShadowSlot = InitSlot;
992 if (IsRDXLiveIn && IsRCXLiveIn)
993 RDXShadowSlot += 8;
994 // Emit the saves if needed.
995 if (IsRCXLiveIn)
996 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
997 RCXShadowSlot)
998 .addReg(X86::RCX);
999 if (IsRDXLiveIn)
1000 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
1001 RDXShadowSlot)
1002 .addReg(X86::RDX);
1003 } else {
1004 // Not in the prolog. Copy RAX to a virtual reg.
1005 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), SizeReg).addReg(X86::RAX);
1006 }
1007
1008 // Add code to MBB to check for overflow and set the new target stack pointer
1009 // to zero if so.
1010 BuildMI(&MBB, DL, TII.get(X86::XOR64rr), ZeroReg)
1011 .addReg(ZeroReg, RegState::Undef)
1012 .addReg(ZeroReg, RegState::Undef);
1013 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), CopyReg).addReg(X86::RSP);
1014 BuildMI(&MBB, DL, TII.get(X86::SUB64rr), TestReg)
1015 .addReg(CopyReg)
1016 .addReg(SizeReg);
1017 BuildMI(&MBB, DL, TII.get(X86::CMOV64rr), FinalReg)
1018 .addReg(TestReg)
1019 .addReg(ZeroReg)
1021
1022 // FinalReg now holds final stack pointer value, or zero if
1023 // allocation would overflow. Compare against the current stack
1024 // limit from the thread environment block. Note this limit is the
1025 // lowest touched page on the stack, not the point at which the OS
1026 // will cause an overflow exception, so this is just an optimization
1027 // to avoid unnecessarily touching pages that are below the current
1028 // SP but already committed to the stack by the OS.
1029 BuildMI(&MBB, DL, TII.get(X86::MOV64rm), LimitReg)
1030 .addReg(0)
1031 .addImm(1)
1032 .addReg(0)
1033 .addImm(ThreadEnvironmentStackLimit)
1034 .addReg(X86::GS);
1035 BuildMI(&MBB, DL, TII.get(X86::CMP64rr)).addReg(FinalReg).addReg(LimitReg);
1036 // Jump if the desired stack pointer is at or above the stack limit.
1037 BuildMI(&MBB, DL, TII.get(X86::JCC_1))
1038 .addMBB(ContinueMBB)
1040
1041 // Add code to roundMBB to round the final stack pointer to a page boundary.
1042 RoundMBB->addLiveIn(FinalReg);
1043 BuildMI(RoundMBB, DL, TII.get(X86::AND64ri32), RoundedReg)
1044 .addReg(FinalReg)
1045 .addImm(PageMask);
1046 BuildMI(RoundMBB, DL, TII.get(X86::JMP_1)).addMBB(LoopMBB);
1047
1048 // LimitReg now holds the current stack limit, RoundedReg page-rounded
1049 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page
1050 // and probe until we reach RoundedReg.
1051 if (!InProlog) {
1052 BuildMI(LoopMBB, DL, TII.get(X86::PHI), JoinReg)
1053 .addReg(LimitReg)
1054 .addMBB(RoundMBB)
1055 .addReg(ProbeReg)
1056 .addMBB(LoopMBB);
1057 }
1058
1059 LoopMBB->addLiveIn(JoinReg);
1060 addRegOffset(BuildMI(LoopMBB, DL, TII.get(X86::LEA64r), ProbeReg), JoinReg,
1061 false, -PageSize);
1062
1063 // Probe by storing a byte onto the stack.
1064 BuildMI(LoopMBB, DL, TII.get(X86::MOV8mi))
1065 .addReg(ProbeReg)
1066 .addImm(1)
1067 .addReg(0)
1068 .addImm(0)
1069 .addReg(0)
1070 .addImm(0);
1071
1072 LoopMBB->addLiveIn(RoundedReg);
1073 BuildMI(LoopMBB, DL, TII.get(X86::CMP64rr))
1074 .addReg(RoundedReg)
1075 .addReg(ProbeReg);
1076 BuildMI(LoopMBB, DL, TII.get(X86::JCC_1))
1077 .addMBB(LoopMBB)
1079
1080 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
1081
1082 // If in prolog, restore RDX and RCX.
1083 if (InProlog) {
1084 if (RCXShadowSlot) // It means we spilled RCX in the prologue.
1085 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL,
1086 TII.get(X86::MOV64rm), X86::RCX),
1087 X86::RSP, false, RCXShadowSlot);
1088 if (RDXShadowSlot) // It means we spilled RDX in the prologue.
1089 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL,
1090 TII.get(X86::MOV64rm), X86::RDX),
1091 X86::RSP, false, RDXShadowSlot);
1092 }
1093
1094 // Now that the probing is done, add code to continueMBB to update
1095 // the stack pointer for real.
1096 ContinueMBB->addLiveIn(SizeReg);
1097 BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::SUB64rr), X86::RSP)
1098 .addReg(X86::RSP)
1099 .addReg(SizeReg);
1100
1101 // Add the control flow edges we need.
1102 MBB.addSuccessor(ContinueMBB);
1103 MBB.addSuccessor(RoundMBB);
1104 RoundMBB->addSuccessor(LoopMBB);
1105 LoopMBB->addSuccessor(ContinueMBB);
1106 LoopMBB->addSuccessor(LoopMBB);
1107
1108 // Mark all the instructions added to the prolog as frame setup.
1109 if (InProlog) {
1110 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
1111 BeforeMBBI->setFlag(MachineInstr::FrameSetup);
1112 }
1113 for (MachineInstr &MI : *RoundMBB) {
1115 }
1116 for (MachineInstr &MI : *LoopMBB) {
1118 }
1119 for (MachineInstr &MI :
1120 llvm::make_range(ContinueMBB->begin(), ContinueMBBI)) {
1122 }
1123 }
1124}
1125
1126void X86FrameLowering::emitStackProbeCall(
1128 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
1129 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
1130 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
1131
1132 // FIXME: Add indirect thunk support and remove this.
1133 if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
1134 report_fatal_error("Emitting stack probe calls on 64-bit with the large "
1135 "code model and indirect thunks not yet implemented.");
1136
1137 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1139 "Stack probe calls will clobber live EFLAGS.");
1140
1141 unsigned CallOp;
1142 if (Is64Bit)
1143 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1144 else
1145 CallOp = X86::CALLpcrel32;
1146
1148
1150 MachineBasicBlock::iterator ExpansionMBBI = std::prev(MBBI);
1151
1152 // All current stack probes take AX and SP as input, clobber flags, and
1153 // preserve all registers. x86_64 probes leave RSP unmodified.
1155 // For the large code model, we have to call through a register. Use R11,
1156 // as it is scratch in all supported calling conventions.
1157 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11)
1159 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11);
1160 } else {
1161 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp))
1163 }
1164
1165 unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX;
1166 unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP;
1172
1173 MachineInstr *ModInst = CI;
1174 if (STI.isTargetWin64() || !STI.isOSWindows()) {
1175 // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves.
1176 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
1177 // themselves. They also does not clobber %rax so we can reuse it when
1178 // adjusting %rsp.
1179 // All other platforms do not specify a particular ABI for the stack probe
1180 // function, so we arbitrarily define it to not adjust %esp/%rsp itself.
1181 ModInst =
1183 .addReg(SP)
1184 .addReg(AX);
1185 }
1186
1187 // DebugInfo variable locations -- if there's an instruction number for the
1188 // allocation (i.e., DYN_ALLOC_*), substitute it for the instruction that
1189 // modifies SP.
1190 if (InstrNum) {
1191 if (STI.isTargetWin64() || !STI.isOSWindows()) {
1192 // Label destination operand of the subtract.
1193 MF.makeDebugValueSubstitution(*InstrNum,
1194 {ModInst->getDebugInstrNum(), 0});
1195 } else {
1196 // Label the call. The operand number is the penultimate operand, zero
1197 // based.
1198 unsigned SPDefOperand = ModInst->getNumOperands() - 2;
1200 *InstrNum, {ModInst->getDebugInstrNum(), SPDefOperand});
1201 }
1202 }
1203
1204 if (InProlog) {
1205 // Apply the frame setup flag to all inserted instrs.
1206 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI)
1207 ExpansionMBBI->setFlag(MachineInstr::FrameSetup);
1208 }
1209}
1210
1211static unsigned calculateSetFPREG(uint64_t SPAdjust) {
1212 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
1213 // and might require smaller successive adjustments.
1214 const uint64_t Win64MaxSEHOffset = 128;
1215 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
1216 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
1217 return SEHFrameOffset & -16;
1218}
1219
1220// If we're forcing a stack realignment we can't rely on just the frame
1221// info, we need to know the ABI stack alignment as well in case we
1222// have a call out. Otherwise just make sure we have some alignment - we'll
1223// go with the minimum SlotSize.
1225X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
1226 const MachineFrameInfo &MFI = MF.getFrameInfo();
1227 Align MaxAlign = MFI.getMaxAlign(); // Desired stack alignment.
1229 bool HasRealign = MF.getFunction().hasFnAttribute("stackrealign");
1230 if (HasRealign) {
1231 if (MFI.hasCalls())
1232 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1233 else if (MaxAlign < SlotSize)
1234 MaxAlign = Align(SlotSize);
1235 }
1236
1238 if (HasRealign)
1239 MaxAlign = (MaxAlign > 16) ? MaxAlign : Align(16);
1240 else
1241 MaxAlign = Align(16);
1242 }
1243 return MaxAlign.value();
1244}
1245
1246void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
1248 const DebugLoc &DL, unsigned Reg,
1249 uint64_t MaxAlign) const {
1250 uint64_t Val = -MaxAlign;
1251 unsigned AndOp = getANDriOpcode(Uses64BitFramePtr, Val);
1252
1253 MachineFunction &MF = *MBB.getParent();
1255 const X86TargetLowering &TLI = *STI.getTargetLowering();
1256 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1257 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1258
1259 // We want to make sure that (in worst case) less than StackProbeSize bytes
1260 // are not probed after the AND. This assumption is used in
1261 // emitStackProbeInlineGeneric.
1262 if (Reg == StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1263 {
1264 NumFrameLoopProbe++;
1265 MachineBasicBlock *entryMBB =
1267 MachineBasicBlock *headMBB =
1269 MachineBasicBlock *bodyMBB =
1271 MachineBasicBlock *footMBB =
1273
1275 MF.insert(MBBIter, entryMBB);
1276 MF.insert(MBBIter, headMBB);
1277 MF.insert(MBBIter, bodyMBB);
1278 MF.insert(MBBIter, footMBB);
1279 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1280 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
1281 : Is64Bit ? X86::R11D
1282 : X86::EAX;
1283
1284 // Setup entry block
1285 {
1286
1287 entryMBB->splice(entryMBB->end(), &MBB, MBB.begin(), MBBI);
1288 BuildMI(entryMBB, DL, TII.get(TargetOpcode::COPY), FinalStackProbed)
1291 MachineInstr *MI =
1292 BuildMI(entryMBB, DL, TII.get(AndOp), FinalStackProbed)
1293 .addReg(FinalStackProbed)
1294 .addImm(Val)
1296
1297 // The EFLAGS implicit def is dead.
1298 MI->getOperand(3).setIsDead();
1299
1300 BuildMI(entryMBB, DL,
1301 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1302 .addReg(FinalStackProbed)
1305 BuildMI(entryMBB, DL, TII.get(X86::JCC_1))
1306 .addMBB(&MBB)
1309 entryMBB->addSuccessor(headMBB);
1310 entryMBB->addSuccessor(&MBB);
1311 }
1312
1313 // Loop entry block
1314
1315 {
1316 const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr);
1317 BuildMI(headMBB, DL, TII.get(SUBOpc), StackPtr)
1319 .addImm(StackProbeSize)
1321
1322 BuildMI(headMBB, DL,
1323 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1325 .addReg(FinalStackProbed)
1327
1328 // jump to the footer if StackPtr < FinalStackProbed
1329 BuildMI(headMBB, DL, TII.get(X86::JCC_1))
1330 .addMBB(footMBB)
1333
1334 headMBB->addSuccessor(bodyMBB);
1335 headMBB->addSuccessor(footMBB);
1336 }
1337
1338 // setup loop body
1339 {
1340 addRegOffset(BuildMI(bodyMBB, DL, TII.get(MovMIOpc))
1342 StackPtr, false, 0)
1343 .addImm(0)
1345
1346 const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr);
1347 BuildMI(bodyMBB, DL, TII.get(SUBOpc), StackPtr)
1349 .addImm(StackProbeSize)
1351
1352 // cmp with stack pointer bound
1353 BuildMI(bodyMBB, DL,
1354 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1355 .addReg(FinalStackProbed)
1358
1359 // jump back while FinalStackProbed < StackPtr
1360 BuildMI(bodyMBB, DL, TII.get(X86::JCC_1))
1361 .addMBB(bodyMBB)
1364 bodyMBB->addSuccessor(bodyMBB);
1365 bodyMBB->addSuccessor(footMBB);
1366 }
1367
1368 // setup loop footer
1369 {
1370 BuildMI(footMBB, DL, TII.get(TargetOpcode::COPY), StackPtr)
1371 .addReg(FinalStackProbed)
1373 addRegOffset(BuildMI(footMBB, DL, TII.get(MovMIOpc))
1375 StackPtr, false, 0)
1376 .addImm(0)
1378 footMBB->addSuccessor(&MBB);
1379 }
1380
1381 recomputeLiveIns(*headMBB);
1382 recomputeLiveIns(*bodyMBB);
1383 recomputeLiveIns(*footMBB);
1385 }
1386 } else {
1387 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg)
1388 .addReg(Reg)
1389 .addImm(Val)
1391
1392 // The EFLAGS implicit def is dead.
1393 MI->getOperand(3).setIsDead();
1394 }
1395}
1396
1398 // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be
1399 // clobbered by any interrupt handler.
1400 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1401 "MF used frame lowering for wrong subtarget");
1402 const Function &Fn = MF.getFunction();
1403 const bool IsWin64CC = STI.isCallingConvWin64(Fn.getCallingConv());
1404 return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Attribute::NoRedZone);
1405}
1406
1407/// Return true if we need to use the restricted Windows x64 prologue and
1408/// epilogue code patterns that can be described with WinCFI (.seh_*
1409/// directives).
1410bool X86FrameLowering::isWin64Prologue(const MachineFunction &MF) const {
1411 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
1412}
1413
1414bool X86FrameLowering::needsDwarfCFI(const MachineFunction &MF) const {
1415 return !isWin64Prologue(MF) && MF.needsFrameMoves();
1416}
1417
1418/// emitPrologue - Push callee-saved registers onto the stack, which
1419/// automatically adjust the stack pointer. Adjust the stack pointer to allocate
1420/// space for local variables. Also emit labels used by the exception handler to
1421/// generate the exception handling frames.
1422
1423/*
1424 Here's a gist of what gets emitted:
1425
1426 ; Establish frame pointer, if needed
1427 [if needs FP]
1428 push %rbp
1429 .cfi_def_cfa_offset 16
1430 .cfi_offset %rbp, -16
1431 .seh_pushreg %rpb
1432 mov %rsp, %rbp
1433 .cfi_def_cfa_register %rbp
1434
1435 ; Spill general-purpose registers
1436 [for all callee-saved GPRs]
1437 pushq %<reg>
1438 [if not needs FP]
1439 .cfi_def_cfa_offset (offset from RETADDR)
1440 .seh_pushreg %<reg>
1441
1442 ; If the required stack alignment > default stack alignment
1443 ; rsp needs to be re-aligned. This creates a "re-alignment gap"
1444 ; of unknown size in the stack frame.
1445 [if stack needs re-alignment]
1446 and $MASK, %rsp
1447
1448 ; Allocate space for locals
1449 [if target is Windows and allocated space > 4096 bytes]
1450 ; Windows needs special care for allocations larger
1451 ; than one page.
1452 mov $NNN, %rax
1453 call ___chkstk_ms/___chkstk
1454 sub %rax, %rsp
1455 [else]
1456 sub $NNN, %rsp
1457
1458 [if needs FP]
1459 .seh_stackalloc (size of XMM spill slots)
1460 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots
1461 [else]
1462 .seh_stackalloc NNN
1463
1464 ; Spill XMMs
1465 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved,
1466 ; they may get spilled on any platform, if the current function
1467 ; calls @llvm.eh.unwind.init
1468 [if needs FP]
1469 [for all callee-saved XMM registers]
1470 movaps %<xmm reg>, -MMM(%rbp)
1471 [for all callee-saved XMM registers]
1472 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset)
1473 ; i.e. the offset relative to (%rbp - SEHFrameOffset)
1474 [else]
1475 [for all callee-saved XMM registers]
1476 movaps %<xmm reg>, KKK(%rsp)
1477 [for all callee-saved XMM registers]
1478 .seh_savexmm %<xmm reg>, KKK
1479
1480 .seh_endprologue
1481
1482 [if needs base pointer]
1483 mov %rsp, %rbx
1484 [if needs to restore base pointer]
1485 mov %rsp, -MMM(%rbp)
1486
1487 ; Emit CFI info
1488 [if needs FP]
1489 [for all callee-saved registers]
1490 .cfi_offset %<reg>, (offset from %rbp)
1491 [else]
1492 .cfi_def_cfa_offset (offset from RETADDR)
1493 [for all callee-saved registers]
1494 .cfi_offset %<reg>, (offset from %rsp)
1495
1496 Notes:
1497 - .seh directives are emitted only for Windows 64 ABI
1498 - .cv_fpo directives are emitted on win32 when emitting CodeView
1499 - .cfi directives are emitted for all other ABIs
1500 - for 32-bit code, substitute %e?? registers for %r??
1501*/
1502
1504 MachineBasicBlock &MBB) const {
1505 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1506 "MF used frame lowering for wrong subtarget");
1508 MachineFrameInfo &MFI = MF.getFrameInfo();
1509 const Function &Fn = MF.getFunction();
1510 MachineModuleInfo &MMI = MF.getMMI();
1512 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
1513 uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate.
1514 bool IsFunclet = MBB.isEHFuncletEntry();
1516 if (Fn.hasPersonalityFn())
1517 Personality = classifyEHPersonality(Fn.getPersonalityFn());
1518 bool FnHasClrFunclet =
1519 MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR;
1520 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1521 bool HasFP = hasFP(MF);
1522 bool IsWin64Prologue = isWin64Prologue(MF);
1523 bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry();
1524 // FIXME: Emit FPO data for EH funclets.
1525 bool NeedsWinFPO =
1526 !IsFunclet && STI.isTargetWin32() && MMI.getModule()->getCodeViewFlag();
1527 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1528 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1530 const Register MachineFramePtr =
1532 : FramePtr;
1533 Register BasePtr = TRI->getBaseRegister();
1534 bool HasWinCFI = false;
1535
1536 // Debug location must be unknown since the first debug location is used
1537 // to determine the end of the prologue.
1538 DebugLoc DL;
1539 Register ArgBaseReg;
1540
1541 // Emit extra prolog for argument stack slot reference.
1542 if (auto *MI = X86FI->getStackPtrSaveMI()) {
1543 // MI is lea instruction that created in X86ArgumentStackSlotPass.
1544 // Creat extra prolog for stack realignment.
1545 ArgBaseReg = MI->getOperand(0).getReg();
1546 // leal 4(%esp), %basereg
1547 // .cfi_def_cfa %basereg, 0
1548 // andl $-128, %esp
1549 // pushl -4(%basereg)
1550 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::LEA64r : X86::LEA32r),
1551 ArgBaseReg)
1553 .addImm(1)
1554 .addUse(X86::NoRegister)
1556 .addUse(X86::NoRegister)
1558 if (NeedsDwarfCFI) {
1559 // .cfi_def_cfa %basereg, 0
1560 unsigned DwarfStackPtr = TRI->getDwarfRegNum(ArgBaseReg, true);
1561 BuildCFI(MBB, MBBI, DL,
1562 MCCFIInstruction::cfiDefCfa(nullptr, DwarfStackPtr, 0),
1564 }
1565 BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign);
1566 int64_t Offset = -(int64_t)SlotSize;
1567 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64rmm : X86::PUSH32rmm))
1568 .addReg(ArgBaseReg)
1569 .addImm(1)
1570 .addReg(X86::NoRegister)
1571 .addImm(Offset)
1572 .addReg(X86::NoRegister)
1574 }
1575
1576 // Space reserved for stack-based arguments when making a (ABI-guaranteed)
1577 // tail call.
1578 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
1579 if (TailCallArgReserveSize && IsWin64Prologue)
1580 report_fatal_error("Can't handle guaranteed tail call under win64 yet");
1581
1582 const bool EmitStackProbeCall =
1584 unsigned StackProbeSize = STI.getTargetLowering()->getStackProbeSize(MF);
1585
1586 if (HasFP && X86FI->hasSwiftAsyncContext()) {
1590 // The special symbol below is absolute and has a *value* suitable to be
1591 // combined with the frame pointer directly.
1592 BuildMI(MBB, MBBI, DL, TII.get(X86::OR64rm), MachineFramePtr)
1593 .addUse(MachineFramePtr)
1594 .addUse(X86::RIP)
1595 .addImm(1)
1596 .addUse(X86::NoRegister)
1597 .addExternalSymbol("swift_async_extendedFramePointerFlags",
1599 .addUse(X86::NoRegister);
1600 break;
1601 }
1602 [[fallthrough]];
1603
1605 BuildMI(MBB, MBBI, DL, TII.get(X86::BTS64ri8), MachineFramePtr)
1606 .addUse(MachineFramePtr)
1607 .addImm(60)
1609 break;
1610
1612 break;
1613 }
1614 }
1615
1616 // Re-align the stack on 64-bit if the x86-interrupt calling convention is
1617 // used and an error code was pushed, since the x86-64 ABI requires a 16-byte
1618 // stack alignment.
1620 Fn.arg_size() == 2) {
1621 StackSize += 8;
1622 MFI.setStackSize(StackSize);
1623
1624 // Update the stack pointer by pushing a register. This is the instruction
1625 // emitted that would be end up being emitted by a call to `emitSPUpdate`.
1626 // Hard-coding the update to a push avoids emitting a second
1627 // `STACKALLOC_W_PROBING` instruction in the save block: We know that stack
1628 // probing isn't needed anyways for an 8-byte update.
1629 // Pushing a register leaves us in a similar situation to a regular
1630 // function call where we know that the address at (rsp-8) is writeable.
1631 // That way we avoid any off-by-ones with stack probing for additional
1632 // stack pointer updates later on.
1633 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1634 .addReg(X86::RAX, RegState::Undef)
1636 }
1637
1638 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
1639 // function, and use up to 128 bytes of stack space, don't have a frame
1640 // pointer, calls, or dynamic alloca then we do not need to adjust the
1641 // stack pointer (we fit in the Red Zone). We also check that we don't
1642 // push and pop from the stack.
1643 if (has128ByteRedZone(MF) && !TRI->hasStackRealignment(MF) &&
1644 !MFI.hasVarSizedObjects() && // No dynamic alloca.
1645 !MFI.adjustsStack() && // No calls.
1646 !EmitStackProbeCall && // No stack probes.
1647 !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop.
1648 !MF.shouldSplitStack()) { // Regular stack
1649 uint64_t MinSize =
1651 if (HasFP)
1652 MinSize += SlotSize;
1653 X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0);
1654 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
1655 MFI.setStackSize(StackSize);
1656 }
1657
1658 // Insert stack pointer adjustment for later moving of return addr. Only
1659 // applies to tail call optimized functions where the callee argument stack
1660 // size is bigger than the callers.
1661 if (TailCallArgReserveSize != 0) {
1662 BuildStackAdjustment(MBB, MBBI, DL, -(int)TailCallArgReserveSize,
1663 /*InEpilogue=*/false)
1665 }
1666
1667 // Mapping for machine moves:
1668 //
1669 // DST: VirtualFP AND
1670 // SRC: VirtualFP => DW_CFA_def_cfa_offset
1671 // ELSE => DW_CFA_def_cfa
1672 //
1673 // SRC: VirtualFP AND
1674 // DST: Register => DW_CFA_def_cfa_register
1675 //
1676 // ELSE
1677 // OFFSET < 0 => DW_CFA_offset_extended_sf
1678 // REG < 64 => DW_CFA_offset + Reg
1679 // ELSE => DW_CFA_offset_extended
1680
1681 uint64_t NumBytes = 0;
1682 int stackGrowth = -SlotSize;
1683
1684 // Find the funclet establisher parameter
1685 Register Establisher = X86::NoRegister;
1686 if (IsClrFunclet)
1687 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX;
1688 else if (IsFunclet)
1689 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX;
1690
1691 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1692 // Immediately spill establisher into the home slot.
1693 // The runtime cares about this.
1694 // MOV64mr %rdx, 16(%rsp)
1695 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1696 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), StackPtr, true, 16)
1697 .addReg(Establisher)
1699 MBB.addLiveIn(Establisher);
1700 }
1701
1702 if (HasFP) {
1703 assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved");
1704
1705 // Calculate required stack adjustment.
1706 uint64_t FrameSize = StackSize - SlotSize;
1707 NumBytes =
1708 FrameSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1709
1710 // Callee-saved registers are pushed on stack before the stack is realigned.
1711 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
1712 NumBytes = alignTo(NumBytes, MaxAlign);
1713
1714 // Save EBP/RBP into the appropriate stack slot.
1715 BuildMI(MBB, MBBI, DL,
1717 .addReg(MachineFramePtr, RegState::Kill)
1719
1720 if (NeedsDwarfCFI && !ArgBaseReg.isValid()) {
1721 // Mark the place where EBP/RBP was saved.
1722 // Define the current CFA rule to use the provided offset.
1723 assert(StackSize);
1724 BuildCFI(MBB, MBBI, DL,
1726 nullptr, -2 * stackGrowth + (int)TailCallArgReserveSize),
1728
1729 // Change the rule for the FramePtr to be an "offset" rule.
1730 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1731 BuildCFI(MBB, MBBI, DL,
1732 MCCFIInstruction::createOffset(nullptr, DwarfFramePtr,
1733 2 * stackGrowth -
1734 (int)TailCallArgReserveSize),
1736 }
1737
1738 if (NeedsWinCFI) {
1739 HasWinCFI = true;
1740 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1743 }
1744
1745 if (!IsFunclet) {
1746 if (X86FI->hasSwiftAsyncContext()) {
1747 const auto &Attrs = MF.getFunction().getAttributes();
1748
1749 // Before we update the live frame pointer we have to ensure there's a
1750 // valid (or null) asynchronous context in its slot just before FP in
1751 // the frame record, so store it now.
1752 if (Attrs.hasAttrSomewhere(Attribute::SwiftAsync)) {
1753 // We have an initial context in r14, store it just before the frame
1754 // pointer.
1755 MBB.addLiveIn(X86::R14);
1756 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1757 .addReg(X86::R14)
1759 } else {
1760 // No initial context, store null so that there's no pointer that
1761 // could be misused.
1762 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64i32))
1763 .addImm(0)
1765 }
1766
1767 if (NeedsWinCFI) {
1768 HasWinCFI = true;
1769 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1770 .addImm(X86::R14)
1772 }
1773
1774 BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr)
1775 .addUse(X86::RSP)
1776 .addImm(1)
1777 .addUse(X86::NoRegister)
1778 .addImm(8)
1779 .addUse(X86::NoRegister)
1781 BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64ri32), X86::RSP)
1782 .addUse(X86::RSP)
1783 .addImm(8)
1785 }
1786
1787 if (!IsWin64Prologue && !IsFunclet) {
1788 // Update EBP with the new base value.
1789 if (!X86FI->hasSwiftAsyncContext())
1790 BuildMI(MBB, MBBI, DL,
1791 TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
1792 FramePtr)
1795
1796 if (NeedsDwarfCFI) {
1797 if (ArgBaseReg.isValid()) {
1798 SmallString<64> CfaExpr;
1799 CfaExpr.push_back(dwarf::DW_CFA_expression);
1800 uint8_t buffer[16];
1801 unsigned DwarfReg = TRI->getDwarfRegNum(MachineFramePtr, true);
1802 CfaExpr.append(buffer, buffer + encodeULEB128(DwarfReg, buffer));
1803 CfaExpr.push_back(2);
1804 CfaExpr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
1805 CfaExpr.push_back(0);
1806 // DW_CFA_expression: reg5 DW_OP_breg5 +0
1807 BuildCFI(MBB, MBBI, DL,
1808 MCCFIInstruction::createEscape(nullptr, CfaExpr.str()),
1810 } else {
1811 // Mark effective beginning of when frame pointer becomes valid.
1812 // Define the current CFA to use the EBP/RBP register.
1813 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1814 BuildCFI(
1815 MBB, MBBI, DL,
1816 MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr),
1818 }
1819 }
1820
1821 if (NeedsWinFPO) {
1822 // .cv_fpo_setframe $FramePtr
1823 HasWinCFI = true;
1824 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
1826 .addImm(0)
1828 }
1829 }
1830 }
1831 } else {
1832 assert(!IsFunclet && "funclets without FPs not yet implemented");
1833 NumBytes =
1834 StackSize - (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1835 }
1836
1837 // Update the offset adjustment, which is mainly used by codeview to translate
1838 // from ESP to VFRAME relative local variable offsets.
1839 if (!IsFunclet) {
1840 if (HasFP && TRI->hasStackRealignment(MF))
1841 MFI.setOffsetAdjustment(-NumBytes);
1842 else
1843 MFI.setOffsetAdjustment(-StackSize);
1844 }
1845
1846 // For EH funclets, only allocate enough space for outgoing calls. Save the
1847 // NumBytes value that we would've used for the parent frame.
1848 unsigned ParentFrameNumBytes = NumBytes;
1849 if (IsFunclet)
1850 NumBytes = getWinEHFuncletFrameSize(MF);
1851
1852 // Skip the callee-saved push instructions.
1853 bool PushedRegs = false;
1854 int StackOffset = 2 * stackGrowth;
1856 auto IsCSPush = [&](const MachineBasicBlock::iterator &MBBI) {
1857 if (MBBI == MBB.end() || !MBBI->getFlag(MachineInstr::FrameSetup))
1858 return false;
1859 unsigned Opc = MBBI->getOpcode();
1860 return Opc == X86::PUSH32r || Opc == X86::PUSH64r || Opc == X86::PUSHP64r ||
1861 Opc == X86::PUSH2 || Opc == X86::PUSH2P;
1862 };
1863
1864 while (IsCSPush(MBBI)) {
1865 PushedRegs = true;
1866 Register Reg = MBBI->getOperand(0).getReg();
1867 LastCSPush = MBBI;
1868 ++MBBI;
1869 unsigned Opc = LastCSPush->getOpcode();
1870
1871 if (!HasFP && NeedsDwarfCFI) {
1872 // Mark callee-saved push instruction.
1873 // Define the current CFA rule to use the provided offset.
1874 assert(StackSize);
1875 // Compared to push, push2 introduces more stack offset (one more
1876 // register).
1877 if (Opc == X86::PUSH2 || Opc == X86::PUSH2P)
1878 StackOffset += stackGrowth;
1879 BuildCFI(MBB, MBBI, DL,
1882 StackOffset += stackGrowth;
1883 }
1884
1885 if (NeedsWinCFI) {
1886 HasWinCFI = true;
1887 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1888 .addImm(Reg)
1890 if (Opc == X86::PUSH2 || Opc == X86::PUSH2P)
1891 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1892 .addImm(LastCSPush->getOperand(1).getReg())
1894 }
1895 }
1896
1897 // Realign stack after we pushed callee-saved registers (so that we'll be
1898 // able to calculate their offsets from the frame pointer).
1899 // Don't do this for Win64, it needs to realign the stack after the prologue.
1900 if (!IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF) &&
1901 !ArgBaseReg.isValid()) {
1902 assert(HasFP && "There should be a frame pointer if stack is realigned.");
1903 BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign);
1904
1905 if (NeedsWinCFI) {
1906 HasWinCFI = true;
1907 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlign))
1908 .addImm(MaxAlign)
1910 }
1911 }
1912
1913 // If there is an SUB32ri of ESP immediately before this instruction, merge
1914 // the two. This can be the case when tail call elimination is enabled and
1915 // the callee has more arguments then the caller.
1916 NumBytes -= mergeSPUpdates(MBB, MBBI, true);
1917
1918 // Adjust stack pointer: ESP -= numbytes.
1919
1920 // Windows and cygwin/mingw require a prologue helper routine when allocating
1921 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw
1922 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the
1923 // stack and adjust the stack pointer in one go. The 64-bit version of
1924 // __chkstk is only responsible for probing the stack. The 64-bit prologue is
1925 // responsible for adjusting the stack pointer. Touching the stack at 4K
1926 // increments is necessary to ensure that the guard pages used by the OS
1927 // virtual memory manager are allocated in correct sequence.
1928 uint64_t AlignedNumBytes = NumBytes;
1929 if (IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF))
1930 AlignedNumBytes = alignTo(AlignedNumBytes, MaxAlign);
1931 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
1932 assert(!X86FI->getUsesRedZone() &&
1933 "The Red Zone is not accounted for in stack probes");
1934
1935 // Check whether EAX is livein for this block.
1936 bool isEAXAlive = isEAXLiveIn(MBB);
1937
1938 if (isEAXAlive) {
1939 if (Is64Bit) {
1940 // Save RAX
1941 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1942 .addReg(X86::RAX, RegState::Kill)
1944 } else {
1945 // Save EAX
1946 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r))
1947 .addReg(X86::EAX, RegState::Kill)
1949 }
1950 }
1951
1952 if (Is64Bit) {
1953 // Handle the 64-bit Windows ABI case where we need to call __chkstk.
1954 // Function prologue is responsible for adjusting the stack pointer.
1955 int64_t Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
1956 BuildMI(MBB, MBBI, DL, TII.get(getMOVriOpcode(Is64Bit, Alloc)), X86::RAX)
1957 .addImm(Alloc)
1959 } else {
1960 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
1961 // We'll also use 4 already allocated bytes for EAX.
1962 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
1963 .addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
1965 }
1966
1967 // Call __chkstk, __chkstk_ms, or __alloca.
1968 emitStackProbe(MF, MBB, MBBI, DL, true);
1969
1970 if (isEAXAlive) {
1971 // Restore RAX/EAX
1973 if (Is64Bit)
1974 MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV64rm), X86::RAX),
1975 StackPtr, false, NumBytes - 8);
1976 else
1977 MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), X86::EAX),
1978 StackPtr, false, NumBytes - 4);
1979 MI->setFlag(MachineInstr::FrameSetup);
1980 MBB.insert(MBBI, MI);
1981 }
1982 } else if (NumBytes) {
1983 emitSPUpdate(MBB, MBBI, DL, -(int64_t)NumBytes, /*InEpilogue=*/false);
1984 }
1985
1986 if (NeedsWinCFI && NumBytes) {
1987 HasWinCFI = true;
1988 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc))
1989 .addImm(NumBytes)
1991 }
1992
1993 int SEHFrameOffset = 0;
1994 unsigned SPOrEstablisher;
1995 if (IsFunclet) {
1996 if (IsClrFunclet) {
1997 // The establisher parameter passed to a CLR funclet is actually a pointer
1998 // to the (mostly empty) frame of its nearest enclosing funclet; we have
1999 // to find the root function establisher frame by loading the PSPSym from
2000 // the intermediate frame.
2001 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2002 MachinePointerInfo NoInfo;
2003 MBB.addLiveIn(Establisher);
2004 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher),
2005 Establisher, false, PSPSlotOffset)
2008 ;
2009 // Save the root establisher back into the current funclet's (mostly
2010 // empty) frame, in case a sub-funclet or the GC needs it.
2011 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr,
2012 false, PSPSlotOffset)
2013 .addReg(Establisher)
2015 NoInfo,
2018 }
2019 SPOrEstablisher = Establisher;
2020 } else {
2021 SPOrEstablisher = StackPtr;
2022 }
2023
2024 if (IsWin64Prologue && HasFP) {
2025 // Set RBP to a small fixed offset from RSP. In the funclet case, we base
2026 // this calculation on the incoming establisher, which holds the value of
2027 // RSP from the parent frame at the end of the prologue.
2028 SEHFrameOffset = calculateSetFPREG(ParentFrameNumBytes);
2029 if (SEHFrameOffset)
2030 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr),
2031 SPOrEstablisher, false, SEHFrameOffset);
2032 else
2033 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr)
2034 .addReg(SPOrEstablisher);
2035
2036 // If this is not a funclet, emit the CFI describing our frame pointer.
2037 if (NeedsWinCFI && !IsFunclet) {
2038 assert(!NeedsWinFPO && "this setframe incompatible with FPO data");
2039 HasWinCFI = true;
2040 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
2042 .addImm(SEHFrameOffset)
2044 if (isAsynchronousEHPersonality(Personality))
2045 MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset;
2046 }
2047 } else if (IsFunclet && STI.is32Bit()) {
2048 // Reset EBP / ESI to something good for funclets.
2050 // If we're a catch funclet, we can be returned to via catchret. Save ESP
2051 // into the registration node so that the runtime will restore it for us.
2052 if (!MBB.isCleanupFuncletEntry()) {
2053 assert(Personality == EHPersonality::MSVC_CXX);
2054 Register FrameReg;
2056 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg).getFixed();
2057 // ESP is the first field, so no extra displacement is needed.
2058 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32mr)), FrameReg,
2059 false, EHRegOffset)
2060 .addReg(X86::ESP);
2061 }
2062 }
2063
2064 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) {
2065 const MachineInstr &FrameInstr = *MBBI;
2066 ++MBBI;
2067
2068 if (NeedsWinCFI) {
2069 int FI;
2070 if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) {
2071 if (X86::FR64RegClass.contains(Reg)) {
2072 int Offset;
2073 Register IgnoredFrameReg;
2074 if (IsWin64Prologue && IsFunclet)
2075 Offset = getWin64EHFrameIndexRef(MF, FI, IgnoredFrameReg);
2076 else
2077 Offset =
2078 getFrameIndexReference(MF, FI, IgnoredFrameReg).getFixed() +
2079 SEHFrameOffset;
2080
2081 HasWinCFI = true;
2082 assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data");
2083 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM))
2084 .addImm(Reg)
2085 .addImm(Offset)
2087 }
2088 }
2089 }
2090 }
2091
2092 if (NeedsWinCFI && HasWinCFI)
2093 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue))
2095
2096 if (FnHasClrFunclet && !IsFunclet) {
2097 // Save the so-called Initial-SP (i.e. the value of the stack pointer
2098 // immediately after the prolog) into the PSPSlot so that funclets
2099 // and the GC can recover it.
2100 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2101 auto PSPInfo = MachinePointerInfo::getFixedStack(
2103 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, false,
2104 PSPSlotOffset)
2109 }
2110
2111 // Realign stack after we spilled callee-saved registers (so that we'll be
2112 // able to calculate their offsets from the frame pointer).
2113 // Win64 requires aligning the stack after the prologue.
2114 if (IsWin64Prologue && TRI->hasStackRealignment(MF)) {
2115 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2116 BuildStackAlignAND(MBB, MBBI, DL, SPOrEstablisher, MaxAlign);
2117 }
2118
2119 // We already dealt with stack realignment and funclets above.
2120 if (IsFunclet && STI.is32Bit())
2121 return;
2122
2123 // If we need a base pointer, set it up here. It's whatever the value
2124 // of the stack pointer is at this point. Any variable size objects
2125 // will be allocated after this, so we can still use the base pointer
2126 // to reference locals.
2127 if (TRI->hasBasePointer(MF)) {
2128 // Update the base pointer with the current stack pointer.
2129 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
2130 BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr)
2131 .addReg(SPOrEstablisher)
2133 if (X86FI->getRestoreBasePointer()) {
2134 // Stash value of base pointer. Saving RSP instead of EBP shortens
2135 // dependence chain. Used by SjLj EH.
2136 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2137 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), FramePtr, true,
2139 .addReg(SPOrEstablisher)
2141 }
2142
2143 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) {
2144 // Stash the value of the frame pointer relative to the base pointer for
2145 // Win32 EH. This supports Win32 EH, which does the inverse of the above:
2146 // it recovers the frame pointer from the base pointer rather than the
2147 // other way around.
2148 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2149 Register UsedReg;
2150 int Offset =
2151 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg)
2152 .getFixed();
2153 assert(UsedReg == BasePtr);
2154 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), UsedReg, true, Offset)
2157 }
2158 }
2159 if (ArgBaseReg.isValid()) {
2160 // Save argument base pointer.
2161 auto *MI = X86FI->getStackPtrSaveMI();
2162 int FI = MI->getOperand(1).getIndex();
2163 unsigned MOVmr = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
2164 // movl %basereg, offset(%ebp)
2165 addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), FI)
2166 .addReg(ArgBaseReg)
2168 }
2169
2170 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2171 // Mark end of stack pointer adjustment.
2172 if (!HasFP && NumBytes) {
2173 // Define the current CFA rule to use the provided offset.
2174 assert(StackSize);
2175 BuildCFI(
2176 MBB, MBBI, DL,
2177 MCCFIInstruction::cfiDefCfaOffset(nullptr, StackSize - stackGrowth),
2179 }
2180
2181 // Emit DWARF info specifying the offsets of the callee-saved registers.
2183 }
2184
2185 // X86 Interrupt handling function cannot assume anything about the direction
2186 // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction
2187 // in each prologue of interrupt handler function.
2188 //
2189 // FIXME: Create "cld" instruction only in these cases:
2190 // 1. The interrupt handling function uses any of the "rep" instructions.
2191 // 2. Interrupt handling function calls another function.
2192 //
2194 BuildMI(MBB, MBBI, DL, TII.get(X86::CLD))
2196
2197 // At this point we know if the function has WinCFI or not.
2198 MF.setHasWinCFI(HasWinCFI);
2199}
2200
2202 const MachineFunction &MF) const {
2203 // We can't use LEA instructions for adjusting the stack pointer if we don't
2204 // have a frame pointer in the Win64 ABI. Only ADD instructions may be used
2205 // to deallocate the stack.
2206 // This means that we can use LEA for SP in two situations:
2207 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
2208 // 2. We *have* a frame pointer which means we are permitted to use LEA.
2209 return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF);
2210}
2211
2213 switch (MI.getOpcode()) {
2214 case X86::CATCHRET:
2215 case X86::CLEANUPRET:
2216 return true;
2217 default:
2218 return false;
2219 }
2220 llvm_unreachable("impossible");
2221}
2222
2223// CLR funclets use a special "Previous Stack Pointer Symbol" slot on the
2224// stack. It holds a pointer to the bottom of the root function frame. The
2225// establisher frame pointer passed to a nested funclet may point to the
2226// (mostly empty) frame of its parent funclet, but it will need to find
2227// the frame of the root function to access locals. To facilitate this,
2228// every funclet copies the pointer to the bottom of the root function
2229// frame into a PSPSym slot in its own (mostly empty) stack frame. Using the
2230// same offset for the PSPSym in the root function frame that's used in the
2231// funclets' frames allows each funclet to dynamically accept any ancestor
2232// frame as its establisher argument (the runtime doesn't guarantee the
2233// immediate parent for some reason lost to history), and also allows the GC,
2234// which uses the PSPSym for some bookkeeping, to find it in any funclet's
2235// frame with only a single offset reported for the entire method.
2236unsigned
2237X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const {
2238 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo();
2239 Register SPReg;
2240 int Offset = getFrameIndexReferencePreferSP(MF, Info.PSPSymFrameIdx, SPReg,
2241 /*IgnoreSPUpdates*/ true)
2242 .getFixed();
2243 assert(Offset >= 0 && SPReg == TRI->getStackRegister());
2244 return static_cast<unsigned>(Offset);
2245}
2246
2247unsigned
2248X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
2250 // This is the size of the pushed CSRs.
2251 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2252 // This is the size of callee saved XMMs.
2253 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2254 unsigned XMMSize =
2255 WinEHXMMSlotInfo.size() * TRI->getSpillSize(X86::VR128RegClass);
2256 // This is the amount of stack a funclet needs to allocate.
2257 unsigned UsedSize;
2258 EHPersonality Personality =
2260 if (Personality == EHPersonality::CoreCLR) {
2261 // CLR funclets need to hold enough space to include the PSPSym, at the
2262 // same offset from the stack pointer (immediately after the prolog) as it
2263 // resides at in the main function.
2264 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize;
2265 } else {
2266 // Other funclets just need enough stack for outgoing call arguments.
2267 UsedSize = MF.getFrameInfo().getMaxCallFrameSize();
2268 }
2269 // RBP is not included in the callee saved register block. After pushing RBP,
2270 // everything is 16 byte aligned. Everything we allocate before an outgoing
2271 // call must also be 16 byte aligned.
2272 unsigned FrameSizeMinusRBP = alignTo(CSSize + UsedSize, getStackAlign());
2273 // Subtract out the size of the callee saved registers. This is how much stack
2274 // each funclet will allocate.
2275 return FrameSizeMinusRBP + XMMSize - CSSize;
2276}
2277
2278static bool isTailCallOpcode(unsigned Opc) {
2279 return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi ||
2280 Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
2281 Opc == X86::TCRETURNdi64 || Opc == X86::TCRETURNmi64;
2282}
2283
2285 MachineBasicBlock &MBB) const {
2286 const MachineFrameInfo &MFI = MF.getFrameInfo();
2289 MachineBasicBlock::iterator MBBI = Terminator;
2290 DebugLoc DL;
2291 if (MBBI != MBB.end())
2292 DL = MBBI->getDebugLoc();
2293 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
2294 const bool Is64BitILP32 = STI.isTarget64BitILP32();
2296 Register MachineFramePtr =
2297 Is64BitILP32 ? Register(getX86SubSuperRegister(FramePtr, 64)) : FramePtr;
2298
2299 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
2300 bool NeedsWin64CFI =
2301 IsWin64Prologue && MF.getFunction().needsUnwindTableEntry();
2302 bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(*MBBI);
2303
2304 // Get the number of bytes to allocate from the FrameInfo.
2305 uint64_t StackSize = MFI.getStackSize();
2306 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2307 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2308 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
2309 bool HasFP = hasFP(MF);
2310 uint64_t NumBytes = 0;
2311
2312 bool NeedsDwarfCFI = (!MF.getTarget().getTargetTriple().isOSDarwin() &&
2314 MF.needsFrameMoves();
2315
2316 Register ArgBaseReg;
2317 if (auto *MI = X86FI->getStackPtrSaveMI()) {
2318 unsigned Opc = X86::LEA32r;
2319 Register StackReg = X86::ESP;
2320 ArgBaseReg = MI->getOperand(0).getReg();
2321 if (STI.is64Bit()) {
2322 Opc = X86::LEA64r;
2323 StackReg = X86::RSP;
2324 }
2325 // leal -4(%basereg), %esp
2326 // .cfi_def_cfa %esp, 4
2327 BuildMI(MBB, MBBI, DL, TII.get(Opc), StackReg)
2328 .addUse(ArgBaseReg)
2329 .addImm(1)
2330 .addUse(X86::NoRegister)
2331 .addImm(-(int64_t)SlotSize)
2332 .addUse(X86::NoRegister)
2334 if (NeedsDwarfCFI) {
2335 unsigned DwarfStackPtr = TRI->getDwarfRegNum(StackReg, true);
2336 BuildCFI(MBB, MBBI, DL,
2337 MCCFIInstruction::cfiDefCfa(nullptr, DwarfStackPtr, SlotSize),
2339 --MBBI;
2340 }
2341 --MBBI;
2342 }
2343
2344 if (IsFunclet) {
2345 assert(HasFP && "EH funclets without FP not yet implemented");
2346 NumBytes = getWinEHFuncletFrameSize(MF);
2347 } else if (HasFP) {
2348 // Calculate required stack adjustment.
2349 uint64_t FrameSize = StackSize - SlotSize;
2350 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2351
2352 // Callee-saved registers were pushed on stack before the stack was
2353 // realigned.
2354 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2355 NumBytes = alignTo(FrameSize, MaxAlign);
2356 } else {
2357 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2358 }
2359 uint64_t SEHStackAllocAmt = NumBytes;
2360
2361 // AfterPop is the position to insert .cfi_restore.
2363 if (HasFP) {
2364 if (X86FI->hasSwiftAsyncContext()) {
2365 // Discard the context.
2366 int Offset = 16 + mergeSPUpdates(MBB, MBBI, true);
2367 emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue*/ true);
2368 }
2369 // Pop EBP.
2370 BuildMI(MBB, MBBI, DL,
2372 MachineFramePtr)
2374
2375 // We need to reset FP to its untagged state on return. Bit 60 is currently
2376 // used to show the presence of an extended frame.
2377 if (X86FI->hasSwiftAsyncContext()) {
2378 BuildMI(MBB, MBBI, DL, TII.get(X86::BTR64ri8), MachineFramePtr)
2379 .addUse(MachineFramePtr)
2380 .addImm(60)
2382 }
2383
2384 if (NeedsDwarfCFI) {
2385 if (!ArgBaseReg.isValid()) {
2386 unsigned DwarfStackPtr =
2387 TRI->getDwarfRegNum(Is64Bit ? X86::RSP : X86::ESP, true);
2388 BuildCFI(MBB, MBBI, DL,
2389 MCCFIInstruction::cfiDefCfa(nullptr, DwarfStackPtr, SlotSize),
2391 }
2392 if (!MBB.succ_empty() && !MBB.isReturnBlock()) {
2393 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
2394 BuildCFI(MBB, AfterPop, DL,
2395 MCCFIInstruction::createRestore(nullptr, DwarfFramePtr),
2397 --MBBI;
2398 --AfterPop;
2399 }
2400 --MBBI;
2401 }
2402 }
2403
2404 MachineBasicBlock::iterator FirstCSPop = MBBI;
2405 // Skip the callee-saved pop instructions.
2406 while (MBBI != MBB.begin()) {
2407 MachineBasicBlock::iterator PI = std::prev(MBBI);
2408 unsigned Opc = PI->getOpcode();
2409
2410 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2411 if (!PI->getFlag(MachineInstr::FrameDestroy) ||
2412 (Opc != X86::POP32r && Opc != X86::POP64r && Opc != X86::BTR64ri8 &&
2413 Opc != X86::ADD64ri32 && Opc != X86::POPP64r && Opc != X86::POP2 &&
2414 Opc != X86::POP2P && Opc != X86::LEA64r))
2415 break;
2416 FirstCSPop = PI;
2417 }
2418
2419 --MBBI;
2420 }
2421 if (ArgBaseReg.isValid()) {
2422 // Restore argument base pointer.
2423 auto *MI = X86FI->getStackPtrSaveMI();
2424 int FI = MI->getOperand(1).getIndex();
2425 unsigned MOVrm = Is64Bit ? X86::MOV64rm : X86::MOV32rm;
2426 // movl offset(%ebp), %basereg
2427 addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(MOVrm), ArgBaseReg), FI)
2429 }
2430 MBBI = FirstCSPop;
2431
2432 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2433 emitCatchRetReturnValue(MBB, FirstCSPop, &*Terminator);
2434
2435 if (MBBI != MBB.end())
2436 DL = MBBI->getDebugLoc();
2437 // If there is an ADD32ri or SUB32ri of ESP immediately before this
2438 // instruction, merge the two instructions.
2439 if (NumBytes || MFI.hasVarSizedObjects())
2440 NumBytes += mergeSPUpdates(MBB, MBBI, true);
2441
2442 // If dynamic alloca is used, then reset esp to point to the last callee-saved
2443 // slot before popping them off! Same applies for the case, when stack was
2444 // realigned. Don't do this if this was a funclet epilogue, since the funclets
2445 // will not do realignment or dynamic stack allocation.
2446 if (((TRI->hasStackRealignment(MF)) || MFI.hasVarSizedObjects()) &&
2447 !IsFunclet) {
2448 if (TRI->hasStackRealignment(MF))
2449 MBBI = FirstCSPop;
2450 unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt);
2451 uint64_t LEAAmount =
2452 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2453
2454 if (X86FI->hasSwiftAsyncContext())
2455 LEAAmount -= 16;
2456
2457 // There are only two legal forms of epilogue:
2458 // - add SEHAllocationSize, %rsp
2459 // - lea SEHAllocationSize(%FramePtr), %rsp
2460 //
2461 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence.
2462 // However, we may use this sequence if we have a frame pointer because the
2463 // effects of the prologue can safely be undone.
2464 if (LEAAmount != 0) {
2465 unsigned Opc = getLEArOpcode(Uses64BitFramePtr);
2467 false, LEAAmount);
2468 --MBBI;
2469 } else {
2470 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr);
2471 BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr).addReg(FramePtr);
2472 --MBBI;
2473 }
2474 } else if (NumBytes) {
2475 // Adjust stack pointer back: ESP += numbytes.
2476 emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true);
2477 if (!HasFP && NeedsDwarfCFI) {
2478 // Define the current CFA rule to use the provided offset.
2479 BuildCFI(MBB, MBBI, DL,
2481 nullptr, CSSize + TailCallArgReserveSize + SlotSize),
2483 }
2484 --MBBI;
2485 }
2486
2487 // Windows unwinder will not invoke function's exception handler if IP is
2488 // either in prologue or in epilogue. This behavior causes a problem when a
2489 // call immediately precedes an epilogue, because the return address points
2490 // into the epilogue. To cope with that, we insert an epilogue marker here,
2491 // then replace it with a 'nop' if it ends up immediately after a CALL in the
2492 // final emitted code.
2493 if (NeedsWin64CFI && MF.hasWinCFI())
2494 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue));
2495
2496 if (!HasFP && NeedsDwarfCFI) {
2497 MBBI = FirstCSPop;
2498 int64_t Offset = -CSSize - SlotSize;
2499 // Mark callee-saved pop instruction.
2500 // Define the current CFA rule to use the provided offset.
2501 while (MBBI != MBB.end()) {
2503 unsigned Opc = PI->getOpcode();
2504 ++MBBI;
2505 if (Opc == X86::POP32r || Opc == X86::POP64r || Opc == X86::POPP64r ||
2506 Opc == X86::POP2 || Opc == X86::POP2P) {
2507 Offset += SlotSize;
2508 // Compared to pop, pop2 introduces more stack offset (one more
2509 // register).
2510 if (Opc == X86::POP2 || Opc == X86::POP2P)
2511 Offset += SlotSize;
2512 BuildCFI(MBB, MBBI, DL,
2515 }
2516 }
2517 }
2518
2519 // Emit DWARF info specifying the restores of the callee-saved registers.
2520 // For epilogue with return inside or being other block without successor,
2521 // no need to generate .cfi_restore for callee-saved registers.
2522 if (NeedsDwarfCFI && !MBB.succ_empty())
2523 emitCalleeSavedFrameMoves(MBB, AfterPop, DL, false);
2524
2525 if (Terminator == MBB.end() || !isTailCallOpcode(Terminator->getOpcode())) {
2526 // Add the return addr area delta back since we are not tail calling.
2527 int Offset = -1 * X86FI->getTCReturnAddrDelta();
2528 assert(Offset >= 0 && "TCDelta should never be positive");
2529 if (Offset) {
2530 // Check for possible merge with preceding ADD instruction.
2531 Offset += mergeSPUpdates(MBB, Terminator, true);
2532 emitSPUpdate(MBB, Terminator, DL, Offset, /*InEpilogue=*/true);
2533 }
2534 }
2535
2536 // Emit tilerelease for AMX kernel.
2537 if (X86FI->hasVirtualTileReg())
2538 BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE));
2539}
2540
2542 int FI,
2543 Register &FrameReg) const {
2544 const MachineFrameInfo &MFI = MF.getFrameInfo();
2545
2546 bool IsFixed = MFI.isFixedObjectIndex(FI);
2547 // We can't calculate offset from frame pointer if the stack is realigned,
2548 // so enforce usage of stack/base pointer. The base pointer is used when we
2549 // have dynamic allocas in addition to dynamic realignment.
2550 if (TRI->hasBasePointer(MF))
2551 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister();
2552 else if (TRI->hasStackRealignment(MF))
2553 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister();
2554 else
2555 FrameReg = TRI->getFrameRegister(MF);
2556
2557 // Offset will hold the offset from the stack pointer at function entry to the
2558 // object.
2559 // We need to factor in additional offsets applied during the prologue to the
2560 // frame, base, and stack pointer depending on which is used.
2563 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2564 uint64_t StackSize = MFI.getStackSize();
2565 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
2566 int64_t FPDelta = 0;
2567
2568 // In an x86 interrupt, remove the offset we added to account for the return
2569 // address from any stack object allocated in the caller's frame. Interrupts
2570 // do not have a standard return address. Fixed objects in the current frame,
2571 // such as SSE register spills, should not get this treatment.
2573 Offset >= 0) {
2575 }
2576
2577 if (IsWin64Prologue) {
2578 assert(!MFI.hasCalls() || (StackSize % 16) == 8);
2579
2580 // Calculate required stack adjustment.
2581 uint64_t FrameSize = StackSize - SlotSize;
2582 // If required, include space for extra hidden slot for stashing base
2583 // pointer.
2584 if (X86FI->getRestoreBasePointer())
2585 FrameSize += SlotSize;
2586 uint64_t NumBytes = FrameSize - CSSize;
2587
2588 uint64_t SEHFrameOffset = calculateSetFPREG(NumBytes);
2589 if (FI && FI == X86FI->getFAIndex())
2590 return StackOffset::getFixed(-SEHFrameOffset);
2591
2592 // FPDelta is the offset from the "traditional" FP location of the old base
2593 // pointer followed by return address and the location required by the
2594 // restricted Win64 prologue.
2595 // Add FPDelta to all offsets below that go through the frame pointer.
2596 FPDelta = FrameSize - SEHFrameOffset;
2597 assert((!MFI.hasCalls() || (FPDelta % 16) == 0) &&
2598 "FPDelta isn't aligned per the Win64 ABI!");
2599 }
2600
2601 if (FrameReg == TRI->getFramePtr()) {
2602 // Skip saved EBP/RBP
2603 Offset += SlotSize;
2604
2605 // Account for restricted Windows prologue.
2606 Offset += FPDelta;
2607
2608 // Skip the RETADDR move area
2609 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
2610 if (TailCallReturnAddrDelta < 0)
2611 Offset -= TailCallReturnAddrDelta;
2612
2614 }
2615
2616 // FrameReg is either the stack pointer or a base pointer. But the base is
2617 // located at the end of the statically known StackSize so the distinction
2618 // doesn't really matter.
2619 if (TRI->hasStackRealignment(MF) || TRI->hasBasePointer(MF))
2620 assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
2621 return StackOffset::getFixed(Offset + StackSize);
2622}
2623
2625 Register &FrameReg) const {
2626 const MachineFrameInfo &MFI = MF.getFrameInfo();
2628 const auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2629 const auto it = WinEHXMMSlotInfo.find(FI);
2630
2631 if (it == WinEHXMMSlotInfo.end())
2632 return getFrameIndexReference(MF, FI, FrameReg).getFixed();
2633
2634 FrameReg = TRI->getStackRegister();
2636 it->second;
2637}
2638
2641 Register &FrameReg,
2642 int Adjustment) const {
2643 const MachineFrameInfo &MFI = MF.getFrameInfo();
2644 FrameReg = TRI->getStackRegister();
2645 return StackOffset::getFixed(MFI.getObjectOffset(FI) -
2646 getOffsetOfLocalArea() + Adjustment);
2647}
2648
2651 int FI, Register &FrameReg,
2652 bool IgnoreSPUpdates) const {
2653
2654 const MachineFrameInfo &MFI = MF.getFrameInfo();
2655 // Does not include any dynamic realign.
2656 const uint64_t StackSize = MFI.getStackSize();
2657 // LLVM arranges the stack as follows:
2658 // ...
2659 // ARG2
2660 // ARG1
2661 // RETADDR
2662 // PUSH RBP <-- RBP points here
2663 // PUSH CSRs
2664 // ~~~~~~~ <-- possible stack realignment (non-win64)
2665 // ...
2666 // STACK OBJECTS
2667 // ... <-- RSP after prologue points here
2668 // ~~~~~~~ <-- possible stack realignment (win64)
2669 //
2670 // if (hasVarSizedObjects()):
2671 // ... <-- "base pointer" (ESI/RBX) points here
2672 // DYNAMIC ALLOCAS
2673 // ... <-- RSP points here
2674 //
2675 // Case 1: In the simple case of no stack realignment and no dynamic
2676 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable
2677 // with fixed offsets from RSP.
2678 //
2679 // Case 2: In the case of stack realignment with no dynamic allocas, fixed
2680 // stack objects are addressed with RBP and regular stack objects with RSP.
2681 //
2682 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used
2683 // to address stack arguments for outgoing calls and nothing else. The "base
2684 // pointer" points to local variables, and RBP points to fixed objects.
2685 //
2686 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the
2687 // answer we give is relative to the SP after the prologue, and not the
2688 // SP in the middle of the function.
2689
2690 if (MFI.isFixedObjectIndex(FI) && TRI->hasStackRealignment(MF) &&
2691 !STI.isTargetWin64())
2692 return getFrameIndexReference(MF, FI, FrameReg);
2693
2694 // If !hasReservedCallFrame the function might have SP adjustement in the
2695 // body. So, even though the offset is statically known, it depends on where
2696 // we are in the function.
2697 if (!IgnoreSPUpdates && !hasReservedCallFrame(MF))
2698 return getFrameIndexReference(MF, FI, FrameReg);
2699
2700 // We don't handle tail calls, and shouldn't be seeing them either.
2702 "we don't handle this case!");
2703
2704 // This is how the math works out:
2705 //
2706 // %rsp grows (i.e. gets lower) left to right. Each box below is
2707 // one word (eight bytes). Obj0 is the stack slot we're trying to
2708 // get to.
2709 //
2710 // ----------------------------------
2711 // | BP | Obj0 | Obj1 | ... | ObjN |
2712 // ----------------------------------
2713 // ^ ^ ^ ^
2714 // A B C E
2715 //
2716 // A is the incoming stack pointer.
2717 // (B - A) is the local area offset (-8 for x86-64) [1]
2718 // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2]
2719 //
2720 // |(E - B)| is the StackSize (absolute value, positive). For a
2721 // stack that grown down, this works out to be (B - E). [3]
2722 //
2723 // E is also the value of %rsp after stack has been set up, and we
2724 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now
2725 // (C - E) == (C - A) - (B - A) + (B - E)
2726 // { Using [1], [2] and [3] above }
2727 // == getObjectOffset - LocalAreaOffset + StackSize
2728
2729 return getFrameIndexReferenceSP(MF, FI, FrameReg, StackSize);
2730}
2731
2734 std::vector<CalleeSavedInfo> &CSI) const {
2735 MachineFrameInfo &MFI = MF.getFrameInfo();
2737
2738 unsigned CalleeSavedFrameSize = 0;
2739 unsigned XMMCalleeSavedFrameSize = 0;
2740 auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2741 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta();
2742
2743 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
2744
2745 if (TailCallReturnAddrDelta < 0) {
2746 // create RETURNADDR area
2747 // arg
2748 // arg
2749 // RETADDR
2750 // { ...
2751 // RETADDR area
2752 // ...
2753 // }
2754 // [EBP]
2755 MFI.CreateFixedObject(-TailCallReturnAddrDelta,
2756 TailCallReturnAddrDelta - SlotSize, true);
2757 }
2758
2759 // Spill the BasePtr if it's used.
2760 if (this->TRI->hasBasePointer(MF)) {
2761 // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
2762 if (MF.hasEHFunclets()) {
2764 X86FI->setHasSEHFramePtrSave(true);
2765 X86FI->setSEHFramePtrSaveIndex(FI);
2766 }
2767 }
2768
2769 if (hasFP(MF)) {
2770 // emitPrologue always spills frame register the first thing.
2771 SpillSlotOffset -= SlotSize;
2772 MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2773
2774 // The async context lives directly before the frame pointer, and we
2775 // allocate a second slot to preserve stack alignment.
2776 if (X86FI->hasSwiftAsyncContext()) {
2777 SpillSlotOffset -= SlotSize;
2778 MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2779 SpillSlotOffset -= SlotSize;
2780 }
2781
2782 // Since emitPrologue and emitEpilogue will handle spilling and restoring of
2783 // the frame register, we can delete it from CSI list and not have to worry
2784 // about avoiding it later.
2785 Register FPReg = TRI->getFrameRegister(MF);
2786 for (unsigned i = 0; i < CSI.size(); ++i) {
2787 if (TRI->regsOverlap(CSI[i].getReg(), FPReg)) {
2788 CSI.erase(CSI.begin() + i);
2789 break;
2790 }
2791 }
2792 }
2793
2794 // Strategy:
2795 // 1. Use push2 when
2796 // a) number of CSR > 1 if no need padding
2797 // b) number of CSR > 2 if need padding
2798 // 2. When the number of CSR push is odd
2799 // a. Start to use push2 from the 1st push if stack is 16B aligned.
2800 // b. Start to use push2 from the 2nd push if stack is not 16B aligned.
2801 // 3. When the number of CSR push is even, start to use push2 from the 1st
2802 // push and make the stack 16B aligned before the push
2803 unsigned NumRegsForPush2 = 0;
2804 if (STI.hasPush2Pop2()) {
2805 unsigned NumCSGPR = llvm::count_if(CSI, [](const CalleeSavedInfo &I) {
2806 return X86::GR64RegClass.contains(I.getReg());
2807 });
2808 bool NeedPadding = (SpillSlotOffset % 16 != 0) && (NumCSGPR % 2 == 0);
2809 bool UsePush2Pop2 = NeedPadding ? NumCSGPR > 2 : NumCSGPR > 1;
2810 X86FI->setPadForPush2Pop2(NeedPadding && UsePush2Pop2);
2811 NumRegsForPush2 = UsePush2Pop2 ? alignDown(NumCSGPR, 2) : 0;
2812 if (X86FI->padForPush2Pop2()) {
2813 SpillSlotOffset -= SlotSize;
2814 MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2815 }
2816 }
2817
2818 // Assign slots for GPRs. It increases frame size.
2819 for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
2820 Register Reg = I.getReg();
2821
2822 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
2823 continue;
2824
2825 // A CSR is a candidate for push2/pop2 when it's slot offset is 16B aligned
2826 // or only an odd number of registers in the candidates.
2827 if (X86FI->getNumCandidatesForPush2Pop2() < NumRegsForPush2 &&
2828 (SpillSlotOffset % 16 == 0 ||
2829 X86FI->getNumCandidatesForPush2Pop2() % 2))
2830 X86FI->addCandidateForPush2Pop2(Reg);
2831
2832 SpillSlotOffset -= SlotSize;
2833 CalleeSavedFrameSize += SlotSize;
2834
2835 int SlotIndex = MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2836 I.setFrameIdx(SlotIndex);
2837 }
2838
2839 // Adjust the offset of spill slot as we know the accurate callee saved frame
2840 // size.
2841 if (X86FI->getRestoreBasePointer()) {
2842 SpillSlotOffset -= SlotSize;
2843 CalleeSavedFrameSize += SlotSize;
2844
2845 MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2846 // TODO: saving the slot index is better?
2847 X86FI->setRestoreBasePointer(CalleeSavedFrameSize);
2848 }
2849 assert(X86FI->getNumCandidatesForPush2Pop2() % 2 == 0 &&
2850 "Expect even candidates for push2/pop2");
2851 if (X86FI->getNumCandidatesForPush2Pop2())
2852 ++NumFunctionUsingPush2Pop2;
2853 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
2854 MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize);
2855
2856 // Assign slots for XMMs.
2857 for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
2858 Register Reg = I.getReg();
2859 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
2860 continue;
2861
2862 // If this is k-register make sure we lookup via the largest legal type.
2863 MVT VT = MVT::Other;
2864 if (X86::VK16RegClass.contains(Reg))
2865 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2866
2867 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2868 unsigned Size = TRI->getSpillSize(*RC);
2869 Align Alignment = TRI->getSpillAlign(*RC);
2870 // ensure alignment
2871 assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86");
2872 SpillSlotOffset = -alignTo(-SpillSlotOffset, Alignment);
2873
2874 // spill into slot
2875 SpillSlotOffset -= Size;
2876 int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SpillSlotOffset);
2877 I.setFrameIdx(SlotIndex);
2878 MFI.ensureMaxAlignment(Alignment);
2879
2880 // Save the start offset and size of XMM in stack frame for funclets.
2881 if (X86::VR128RegClass.contains(Reg)) {
2882 WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize;
2883 XMMCalleeSavedFrameSize += Size;
2884 }
2885 }
2886
2887 return true;
2888}
2889
2894
2895 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI
2896 // for us, and there are no XMM CSRs on Win32.
2897 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows())
2898 return true;
2899
2900 // Push GPRs. It increases frame size.
2901 const MachineFunction &MF = *MBB.getParent();
2903 if (X86FI->padForPush2Pop2())
2904 emitSPUpdate(MBB, MI, DL, -(int64_t)SlotSize, /*InEpilogue=*/false);
2905
2906 // Update LiveIn of the basic block and decide whether we can add a kill flag
2907 // to the use.
2908 auto UpdateLiveInCheckCanKill = [&](Register Reg) {
2909 const MachineRegisterInfo &MRI = MF.getRegInfo();
2910 // Do not set a kill flag on values that are also marked as live-in. This
2911 // happens with the @llvm-returnaddress intrinsic and with arguments
2912 // passed in callee saved registers.
2913 // Omitting the kill flags is conservatively correct even if the live-in
2914 // is not used after all.
2915 if (MRI.isLiveIn(Reg))
2916 return false;
2917 MBB.addLiveIn(Reg);
2918 // Check if any subregister is live-in
2919 for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg)
2920 if (MRI.isLiveIn(*AReg))
2921 return false;
2922 return true;
2923 };
2924 auto UpdateLiveInGetKillRegState = [&](Register Reg) {
2925 return getKillRegState(UpdateLiveInCheckCanKill(Reg));
2926 };
2927
2928 for (auto RI = CSI.rbegin(), RE = CSI.rend(); RI != RE; ++RI) {
2929 Register Reg = RI->getReg();
2930 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
2931 continue;
2932
2933 if (X86FI->isCandidateForPush2Pop2(Reg)) {
2934 Register Reg2 = (++RI)->getReg();
2936 .addReg(Reg, UpdateLiveInGetKillRegState(Reg))
2937 .addReg(Reg2, UpdateLiveInGetKillRegState(Reg2))
2939 } else {
2940 BuildMI(MBB, MI, DL, TII.get(getPUSHOpcode(STI)))
2941 .addReg(Reg, UpdateLiveInGetKillRegState(Reg))
2943 }
2944 }
2945
2946 if (X86FI->getRestoreBasePointer()) {
2947 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2948 Register BaseReg = this->TRI->getBaseRegister();
2949 BuildMI(MBB, MI, DL, TII.get(Opc))
2950 .addReg(BaseReg, getKillRegState(true))
2952 }
2953
2954 // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
2955 // It can be done by spilling XMMs to stack frame.
2956 for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
2957 Register Reg = I.getReg();
2958 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
2959 continue;
2960
2961 // If this is k-register make sure we lookup via the largest legal type.
2962 MVT VT = MVT::Other;
2963 if (X86::VK16RegClass.contains(Reg))
2964 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2965
2966 // Add the callee-saved register as live-in. It's killed at the spill.
2967 MBB.addLiveIn(Reg);
2968 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2969
2970 TII.storeRegToStackSlot(MBB, MI, Reg, true, I.getFrameIdx(), RC, TRI,
2971 Register());
2972 --MI;
2973 MI->setFlag(MachineInstr::FrameSetup);
2974 ++MI;
2975 }
2976
2977 return true;
2978}
2979
2980void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB,
2982 MachineInstr *CatchRet) const {
2983 // SEH shouldn't use catchret.
2986 "SEH should not use CATCHRET");
2987 const DebugLoc &DL = CatchRet->getDebugLoc();
2988 MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(0).getMBB();
2989
2990 // Fill EAX/RAX with the address of the target block.
2991 if (STI.is64Bit()) {
2992 // LEA64r CatchRetTarget(%rip), %rax
2993 BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), X86::RAX)
2994 .addReg(X86::RIP)
2995 .addImm(0)
2996 .addReg(0)
2997 .addMBB(CatchRetTarget)
2998 .addReg(0);
2999 } else {
3000 // MOV32ri $CatchRetTarget, %eax
3001 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
3002 .addMBB(CatchRetTarget);
3003 }
3004
3005 // Record that we've taken the address of CatchRetTarget and no longer just
3006 // reference it in a terminator.
3007 CatchRetTarget->setMachineBlockAddressTaken();
3008}
3009
3013 if (CSI.empty())
3014 return false;
3015
3016 if (MI != MBB.end() && isFuncletReturnInstr(*MI) && STI.isOSWindows()) {
3017 // Don't restore CSRs in 32-bit EH funclets. Matches
3018 // spillCalleeSavedRegisters.
3019 if (STI.is32Bit())
3020 return true;
3021 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form
3022 // funclets. emitEpilogue transforms these to normal jumps.
3023 if (MI->getOpcode() == X86::CATCHRET) {
3024 const Function &F = MBB.getParent()->getFunction();
3025 bool IsSEH = isAsynchronousEHPersonality(
3026 classifyEHPersonality(F.getPersonalityFn()));
3027 if (IsSEH)
3028 return true;
3029 }
3030 }
3031
3033
3034 // Reload XMMs from stack frame.
3035 for (const CalleeSavedInfo &I : CSI) {
3036 Register Reg = I.getReg();
3037 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
3038 continue;
3039
3040 // If this is k-register make sure we lookup via the largest legal type.
3041 MVT VT = MVT::Other;
3042 if (X86::VK16RegClass.contains(Reg))
3043 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
3044
3045 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
3046 TII.loadRegFromStackSlot(MBB, MI, Reg, I.getFrameIdx(), RC, TRI,
3047 Register());
3048 }
3049
3050 // Clear the stack slot for spill base pointer register.
3051 MachineFunction &MF = *MBB.getParent();
3053 if (X86FI->getRestoreBasePointer()) {
3054 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
3055 Register BaseReg = this->TRI->getBaseRegister();
3056 BuildMI(MBB, MI, DL, TII.get(Opc), BaseReg)
3058 }
3059
3060 // POP GPRs.
3061 for (auto I = CSI.begin(), E = CSI.end(); I != E; ++I) {
3062 Register Reg = I->getReg();
3063 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
3064 continue;
3065
3066 if (X86FI->isCandidateForPush2Pop2(Reg))
3067 BuildMI(MBB, MI, DL, TII.get(getPOP2Opcode(STI)), Reg)
3070 else
3071 BuildMI(MBB, MI, DL, TII.get(getPOPOpcode(STI)), Reg)
3073 }
3074 if (X86FI->padForPush2Pop2())
3075 emitSPUpdate(MBB, MI, DL, SlotSize, /*InEpilogue=*/true);
3076
3077 return true;
3078}
3079
3081 BitVector &SavedRegs,
3082 RegScavenger *RS) const {
3084
3085 // Spill the BasePtr if it's used.
3086 if (TRI->hasBasePointer(MF)) {
3087 Register BasePtr = TRI->getBaseRegister();
3088 if (STI.isTarget64BitILP32())
3089 BasePtr = getX86SubSuperRegister(BasePtr, 64);
3090 SavedRegs.set(BasePtr);
3091 }
3092}
3093
3094static bool HasNestArgument(const MachineFunction *MF) {
3095 const Function &F = MF->getFunction();
3096 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
3097 I++) {
3098 if (I->hasNestAttr() && !I->use_empty())
3099 return true;
3100 }
3101 return false;
3102}
3103
3104/// GetScratchRegister - Get a temp register for performing work in the
3105/// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
3106/// and the properties of the function either one or two registers will be
3107/// needed. Set primary to true for the first register, false for the second.
3108static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64,
3109 const MachineFunction &MF, bool Primary) {
3111
3112 // Erlang stuff.
3114 if (Is64Bit)
3115 return Primary ? X86::R14 : X86::R13;
3116 else
3117 return Primary ? X86::EBX : X86::EDI;
3118 }
3119
3120 if (Is64Bit) {
3121 if (IsLP64)
3122 return Primary ? X86::R11 : X86::R12;
3123 else
3124 return Primary ? X86::R11D : X86::R12D;
3125 }
3126
3127 bool IsNested = HasNestArgument(&MF);
3128
3132 if (IsNested)
3133 report_fatal_error("Segmented stacks does not support fastcall with "
3134 "nested function.");
3135 return Primary ? X86::EAX : X86::ECX;
3136 }
3137 if (IsNested)
3138 return Primary ? X86::EDX : X86::EAX;
3139 return Primary ? X86::ECX : X86::EAX;
3140}
3141
3142// The stack limit in the TCB is set to this many bytes above the actual stack
3143// limit.
3145
3147 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3148 MachineFrameInfo &MFI = MF.getFrameInfo();
3149 uint64_t StackSize;
3150 unsigned TlsReg, TlsOffset;
3151 DebugLoc DL;
3152
3153 // To support shrink-wrapping we would need to insert the new blocks
3154 // at the right place and update the branches to PrologueMBB.
3155 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3156
3157 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3158 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3159 "Scratch register is live-in");
3160
3161 if (MF.getFunction().isVarArg())
3162 report_fatal_error("Segmented stacks do not support vararg functions.");
3163 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() &&
3166 report_fatal_error("Segmented stacks not supported on this platform.");
3167
3168 // Eventually StackSize will be calculated by a link-time pass; which will
3169 // also decide whether checking code needs to be injected into this particular
3170 // prologue.
3171 StackSize = MFI.getStackSize();
3172
3173 if (!MFI.needsSplitStackProlog())
3174 return;
3175
3179 bool IsNested = false;
3180
3181 // We need to know if the function has a nest argument only in 64 bit mode.
3182 if (Is64Bit)
3183 IsNested = HasNestArgument(&MF);
3184
3185 // The MOV R10, RAX needs to be in a different block, since the RET we emit in
3186 // allocMBB needs to be last (terminating) instruction.
3187
3188 for (const auto &LI : PrologueMBB.liveins()) {
3189 allocMBB->addLiveIn(LI);
3190 checkMBB->addLiveIn(LI);
3191 }
3192
3193 if (IsNested)
3194 allocMBB->addLiveIn(IsLP64 ? X86::R10 : X86::R10D);
3195
3196 MF.push_front(allocMBB);
3197 MF.push_front(checkMBB);
3198
3199 // When the frame size is less than 256 we just compare the stack
3200 // boundary directly to the value of the stack pointer, per gcc.
3201 bool CompareStackPointer = StackSize < kSplitStackAvailable;
3202
3203 // Read the limit off the current stacklet off the stack_guard location.
3204 if (Is64Bit) {
3205 if (STI.isTargetLinux()) {
3206 TlsReg = X86::FS;
3207 TlsOffset = IsLP64 ? 0x70 : 0x40;
3208 } else if (STI.isTargetDarwin()) {
3209 TlsReg = X86::GS;
3210 TlsOffset = 0x60 + 90 * 8; // See pthread_machdep.h. Steal TLS slot 90.
3211 } else if (STI.isTargetWin64()) {
3212 TlsReg = X86::GS;
3213 TlsOffset = 0x28; // pvArbitrary, reserved for application use
3214 } else if (STI.isTargetFreeBSD()) {
3215 TlsReg = X86::FS;
3216 TlsOffset = 0x18;
3217 } else if (STI.isTargetDragonFly()) {
3218 TlsReg = X86::FS;
3219 TlsOffset = 0x20; // use tls_tcb.tcb_segstack
3220 } else {
3221 report_fatal_error("Segmented stacks not supported on this platform.");
3222 }
3223
3224 if (CompareStackPointer)
3225 ScratchReg = IsLP64 ? X86::RSP : X86::ESP;
3226 else
3227 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::LEA64r : X86::LEA64_32r),
3228 ScratchReg)
3229 .addReg(X86::RSP)
3230 .addImm(1)
3231 .addReg(0)
3232 .addImm(-StackSize)
3233 .addReg(0);
3234
3235 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::CMP64rm : X86::CMP32rm))
3236 .addReg(ScratchReg)
3237 .addReg(0)
3238 .addImm(1)
3239 .addReg(0)
3240 .addImm(TlsOffset)
3241 .addReg(TlsReg);
3242 } else {
3243 if (STI.isTargetLinux()) {
3244 TlsReg = X86::GS;
3245 TlsOffset = 0x30;
3246 } else if (STI.isTargetDarwin()) {
3247 TlsReg = X86::GS;
3248 TlsOffset = 0x48 + 90 * 4;
3249 } else if (STI.isTargetWin32()) {
3250 TlsReg = X86::FS;
3251 TlsOffset = 0x14; // pvArbitrary, reserved for application use
3252 } else if (STI.isTargetDragonFly()) {
3253 TlsReg = X86::FS;
3254 TlsOffset = 0x10; // use tls_tcb.tcb_segstack
3255 } else if (STI.isTargetFreeBSD()) {
3256 report_fatal_error("Segmented stacks not supported on FreeBSD i386.");
3257 } else {
3258 report_fatal_error("Segmented stacks not supported on this platform.");
3259 }
3260
3261 if (CompareStackPointer)
3262 ScratchReg = X86::ESP;
3263 else
3264 BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg)
3265 .addReg(X86::ESP)
3266 .addImm(1)
3267 .addReg(0)
3268 .addImm(-StackSize)
3269 .addReg(0);
3270
3273 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm))
3274 .addReg(ScratchReg)
3275 .addReg(0)
3276 .addImm(0)
3277 .addReg(0)
3278 .addImm(TlsOffset)
3279 .addReg(TlsReg);
3280 } else if (STI.isTargetDarwin()) {
3281
3282 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register.
3283 unsigned ScratchReg2;
3284 bool SaveScratch2;
3285 if (CompareStackPointer) {
3286 // The primary scratch register is available for holding the TLS offset.
3287 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3288 SaveScratch2 = false;
3289 } else {
3290 // Need to use a second register to hold the TLS offset
3291 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, false);
3292
3293 // Unfortunately, with fastcc the second scratch register may hold an
3294 // argument.
3295 SaveScratch2 = MF.getRegInfo().isLiveIn(ScratchReg2);
3296 }
3297
3298 // If Scratch2 is live-in then it needs to be saved.
3299 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
3300 "Scratch register is live-in and not saved");
3301
3302 if (SaveScratch2)
3303 BuildMI(checkMBB, DL, TII.get(X86::PUSH32r))
3304 .addReg(ScratchReg2, RegState::Kill);
3305
3306 BuildMI(checkMBB, DL, TII.get(X86::MOV32ri), ScratchReg2)
3307 .addImm(TlsOffset);
3308 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm))
3309 .addReg(ScratchReg)
3310 .addReg(ScratchReg2)
3311 .addImm(1)
3312 .addReg(0)
3313 .addImm(0)
3314 .addReg(TlsReg);
3315
3316 if (SaveScratch2)
3317 BuildMI(checkMBB, DL, TII.get(X86::POP32r), ScratchReg2);
3318 }
3319 }
3320
3321 // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
3322 // It jumps to normal execution of the function body.
3323 BuildMI(checkMBB, DL, TII.get(X86::JCC_1))
3324 .addMBB(&PrologueMBB)
3326
3327 // On 32 bit we first push the arguments size and then the frame size. On 64
3328 // bit, we pass the stack frame size in r10 and the argument size in r11.
3329 if (Is64Bit) {
3330 // Functions with nested arguments use R10, so it needs to be saved across
3331 // the call to _morestack
3332
3333 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX;
3334 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D;
3335 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D;
3336 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3337
3338 if (IsNested)
3339 BuildMI(allocMBB, DL, TII.get(MOVrr), RegAX).addReg(Reg10);
3340
3341 BuildMI(allocMBB, DL, TII.get(getMOVriOpcode(IsLP64, StackSize)), Reg10)
3342 .addImm(StackSize);
3343 BuildMI(allocMBB, DL,
3345 Reg11)
3346 .addImm(X86FI->getArgumentStackSize());
3347 } else {
3348 BuildMI(allocMBB, DL, TII.get(X86::PUSH32i))
3349 .addImm(X86FI->getArgumentStackSize());
3350 BuildMI(allocMBB, DL, TII.get(X86::PUSH32i)).addImm(StackSize);
3351 }
3352
3353 // __morestack is in libgcc
3355 // Under the large code model, we cannot assume that __morestack lives
3356 // within 2^31 bytes of the call site, so we cannot use pc-relative
3357 // addressing. We cannot perform the call via a temporary register,
3358 // as the rax register may be used to store the static chain, and all
3359 // other suitable registers may be either callee-save or used for
3360 // parameter passing. We cannot use the stack at this point either
3361 // because __morestack manipulates the stack directly.
3362 //
3363 // To avoid these issues, perform an indirect call via a read-only memory
3364 // location containing the address.
3365 //
3366 // This solution is not perfect, as it assumes that the .rodata section
3367 // is laid out within 2^31 bytes of each function body, but this seems
3368 // to be sufficient for JIT.
3369 // FIXME: Add retpoline support and remove the error here..
3371 report_fatal_error("Emitting morestack calls on 64-bit with the large "
3372 "code model and thunks not yet implemented.");
3373 BuildMI(allocMBB, DL, TII.get(X86::CALL64m))
3374 .addReg(X86::RIP)
3375 .addImm(0)
3376 .addReg(0)
3377 .addExternalSymbol("__morestack_addr")
3378 .addReg(0);
3379 } else {
3380 if (Is64Bit)
3381 BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32))
3382 .addExternalSymbol("__morestack");
3383 else
3384 BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32))
3385 .addExternalSymbol("__morestack");
3386 }
3387
3388 if (IsNested)
3389 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET_RESTORE_R10));
3390 else
3391 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET));
3392
3393 allocMBB->addSuccessor(&PrologueMBB);
3394
3395 checkMBB->addSuccessor(allocMBB, BranchProbability::getZero());
3396 checkMBB->addSuccessor(&PrologueMBB, BranchProbability::getOne());
3397
3398#ifdef EXPENSIVE_CHECKS
3399 MF.verify();
3400#endif
3401}
3402
3403/// Lookup an ERTS parameter in the !hipe.literals named metadata node.
3404/// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets
3405/// to fields it needs, through a named metadata node "hipe.literals" containing
3406/// name-value pairs.
3407static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD,
3408 const StringRef LiteralName) {
3409 for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) {
3410 MDNode *Node = HiPELiteralsMD->getOperand(i);
3411 if (Node->getNumOperands() != 2)
3412 continue;
3413 MDString *NodeName = dyn_cast<MDString>(Node->getOperand(0));
3414 ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Node->getOperand(1));
3415 if (!NodeName || !NodeVal)
3416 continue;
3417 ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(NodeVal->getValue());
3418 if (ValConst && NodeName->getString() == LiteralName) {
3419 return ValConst->getZExtValue();
3420 }
3421 }
3422
3423 report_fatal_error("HiPE literal " + LiteralName +
3424 " required but not provided");
3425}
3426
3427// Return true if there are no non-ehpad successors to MBB and there are no
3428// non-meta instructions between MBBI and MBB.end().
3431 return llvm::all_of(
3432 MBB.successors(),
3433 [](const MachineBasicBlock *Succ) { return Succ->isEHPad(); }) &&
3434 std::all_of(MBBI, MBB.end(), [](const MachineInstr &MI) {
3435 return MI.isMetaInstruction();
3436 });
3437}
3438
3439/// Erlang programs may need a special prologue to handle the stack size they
3440/// might need at runtime. That is because Erlang/OTP does not implement a C
3441/// stack but uses a custom implementation of hybrid stack/heap architecture.
3442/// (for more information see Eric Stenman's Ph.D. thesis:
3443/// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
3444///
3445/// CheckStack:
3446/// temp0 = sp - MaxStack
3447/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3448/// OldStart:
3449/// ...
3450/// IncStack:
3451/// call inc_stack # doubles the stack space
3452/// temp0 = sp - MaxStack
3453/// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3455 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3456 MachineFrameInfo &MFI = MF.getFrameInfo();
3457 DebugLoc DL;
3458
3459 // To support shrink-wrapping we would need to insert the new blocks
3460 // at the right place and update the branches to PrologueMBB.
3461 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3462
3463 // HiPE-specific values
3464 NamedMDNode *HiPELiteralsMD =
3465 MF.getMMI().getModule()->getNamedMetadata("hipe.literals");
3466 if (!HiPELiteralsMD)
3468 "Can't generate HiPE prologue without runtime parameters");
3469 const unsigned HipeLeafWords = getHiPELiteral(
3470 HiPELiteralsMD, Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS");
3471 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
3472 const unsigned Guaranteed = HipeLeafWords * SlotSize;
3473 unsigned CallerStkArity = MF.getFunction().arg_size() > CCRegisteredArgs
3474 ? MF.getFunction().arg_size() - CCRegisteredArgs
3475 : 0;
3476 unsigned MaxStack = MFI.getStackSize() + CallerStkArity * SlotSize + SlotSize;
3477
3479 "HiPE prologue is only supported on Linux operating systems.");
3480
3481 // Compute the largest caller's frame that is needed to fit the callees'
3482 // frames. This 'MaxStack' is computed from:
3483 //
3484 // a) the fixed frame size, which is the space needed for all spilled temps,
3485 // b) outgoing on-stack parameter areas, and
3486 // c) the minimum stack space this function needs to make available for the
3487 // functions it calls (a tunable ABI property).
3488 if (MFI.hasCalls()) {
3489 unsigned MoreStackForCalls = 0;
3490
3491 for (auto &MBB : MF) {
3492 for (auto &MI : MBB) {
3493 if (!MI.isCall())
3494 continue;
3495
3496 // Get callee operand.
3497 const MachineOperand &MO = MI.getOperand(0);
3498
3499 // Only take account of global function calls (no closures etc.).
3500 if (!MO.isGlobal())
3501 continue;
3502
3503 const Function *F = dyn_cast<Function>(MO.getGlobal());
3504 if (!F)
3505 continue;
3506
3507 // Do not update 'MaxStack' for primitive and built-in functions
3508 // (encoded with names either starting with "erlang."/"bif_" or not
3509 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
3510 // "_", such as the BIF "suspend_0") as they are executed on another
3511 // stack.
3512 if (F->getName().contains("erlang.") || F->getName().contains("bif_") ||
3513 F->getName().find_first_of("._") == StringRef::npos)
3514 continue;
3515
3516 unsigned CalleeStkArity = F->arg_size() > CCRegisteredArgs
3517 ? F->arg_size() - CCRegisteredArgs
3518 : 0;
3519 if (HipeLeafWords - 1 > CalleeStkArity)
3520 MoreStackForCalls =
3521 std::max(MoreStackForCalls,
3522 (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
3523 }
3524 }
3525 MaxStack += MoreStackForCalls;
3526 }
3527
3528 // If the stack frame needed is larger than the guaranteed then runtime checks
3529 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
3530 if (MaxStack > Guaranteed) {
3531 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
3532 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
3533
3534 for (const auto &LI : PrologueMBB.liveins()) {
3535 stackCheckMBB->addLiveIn(LI);
3536 incStackMBB->addLiveIn(LI);
3537 }
3538
3539 MF.push_front(incStackMBB);
3540 MF.push_front(stackCheckMBB);
3541
3542 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3543 unsigned LEAop, CMPop, CALLop;
3544 SPLimitOffset = getHiPELiteral(HiPELiteralsMD, "P_NSP_LIMIT");
3545 if (Is64Bit) {
3546 SPReg = X86::RSP;
3547 PReg = X86::RBP;
3548 LEAop = X86::LEA64r;
3549 CMPop = X86::CMP64rm;
3550 CALLop = X86::CALL64pcrel32;
3551 } else {
3552 SPReg = X86::ESP;
3553 PReg = X86::EBP;
3554 LEAop = X86::LEA32r;
3555 CMPop = X86::CMP32rm;
3556 CALLop = X86::CALLpcrel32;
3557 }
3558
3559 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3560 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3561 "HiPE prologue scratch register is live-in");
3562
3563 // Create new MBB for StackCheck:
3564 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(LEAop), ScratchReg), SPReg,
3565 false, -MaxStack);
3566 // SPLimitOffset is in a fixed heap location (pointed by BP).
3567 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop)).addReg(ScratchReg),
3568 PReg, false, SPLimitOffset);
3569 BuildMI(stackCheckMBB, DL, TII.get(X86::JCC_1))
3570 .addMBB(&PrologueMBB)
3572
3573 // Create new MBB for IncStack:
3574 BuildMI(incStackMBB, DL, TII.get(CALLop)).addExternalSymbol("inc_stack_0");
3575 addRegOffset(BuildMI(incStackMBB, DL, TII.get(LEAop), ScratchReg), SPReg,
3576 false, -MaxStack);
3577 addRegOffset(BuildMI(incStackMBB, DL, TII.get(CMPop)).addReg(ScratchReg),
3578 PReg, false, SPLimitOffset);
3579 BuildMI(incStackMBB, DL, TII.get(X86::JCC_1))
3580 .addMBB(incStackMBB)
3582
3583 stackCheckMBB->addSuccessor(&PrologueMBB, {99, 100});
3584 stackCheckMBB->addSuccessor(incStackMBB, {1, 100});
3585 incStackMBB->addSuccessor(&PrologueMBB, {99, 100});
3586 incStackMBB->addSuccessor(incStackMBB, {1, 100});
3587 }
3588#ifdef EXPENSIVE_CHECKS
3589 MF.verify();
3590#endif
3591}
3592
3593bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB,
3595 const DebugLoc &DL,
3596 int Offset) const {
3597 if (Offset <= 0)
3598 return false;
3599
3600 if (Offset % SlotSize)
3601 return false;
3602
3603 int NumPops = Offset / SlotSize;
3604 // This is only worth it if we have at most 2 pops.
3605 if (NumPops != 1 && NumPops != 2)
3606 return false;
3607
3608 // Handle only the trivial case where the adjustment directly follows
3609 // a call. This is the most common one, anyway.
3610 if (MBBI == MBB.begin())
3611 return false;
3612 MachineBasicBlock::iterator Prev = std::prev(MBBI);
3613 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask())
3614 return false;
3615
3616 unsigned Regs[2];
3617 unsigned FoundRegs = 0;
3618
3620 const MachineOperand &RegMask = Prev->getOperand(1);
3621
3622 auto &RegClass =
3623 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
3624 // Try to find up to NumPops free registers.
3625 for (auto Candidate : RegClass) {
3626 // Poor man's liveness:
3627 // Since we're immediately after a call, any register that is clobbered
3628 // by the call and not defined by it can be considered dead.
3629 if (!RegMask.clobbersPhysReg(Candidate))
3630 continue;
3631
3632 // Don't clobber reserved registers
3633 if (MRI.isReserved(Candidate))
3634 continue;
3635
3636 bool IsDef = false;
3637 for (const MachineOperand &MO : Prev->implicit_operands()) {
3638 if (MO.isReg() && MO.isDef() &&
3639 TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) {
3640 IsDef = true;
3641 break;
3642 }
3643 }
3644
3645 if (IsDef)
3646 continue;
3647
3648 Regs[FoundRegs++] = Candidate;
3649 if (FoundRegs == (unsigned)NumPops)
3650 break;
3651 }
3652
3653 if (FoundRegs == 0)
3654 return false;
3655
3656 // If we found only one free register, but need two, reuse the same one twice.
3657 while (FoundRegs < (unsigned)NumPops)
3658 Regs[FoundRegs++] = Regs[0];
3659
3660 for (int i = 0; i < NumPops; ++i)
3661 BuildMI(MBB, MBBI, DL, TII.get(STI.is64Bit() ? X86::POP64r : X86::POP32r),
3662 Regs[i]);
3663
3664 return true;
3665}
3666
3670 bool reserveCallFrame = hasReservedCallFrame(MF);
3671 unsigned Opcode = I->getOpcode();
3672 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
3673 DebugLoc DL = I->getDebugLoc(); // copy DebugLoc as I will be erased.
3674 uint64_t Amount = TII.getFrameSize(*I);
3675 uint64_t InternalAmt = (isDestroy || Amount) ? TII.getFrameAdjustment(*I) : 0;
3676 I = MBB.erase(I);
3677 auto InsertPos = skipDebugInstructionsForward(I, MBB.end());
3678
3679 // Try to avoid emitting dead SP adjustments if the block end is unreachable,
3680 // typically because the function is marked noreturn (abort, throw,
3681 // assert_fail, etc).
3682 if (isDestroy && blockEndIsUnreachable(MBB, I))
3683 return I;
3684
3685 if (!reserveCallFrame) {
3686 // If the stack pointer can be changed after prologue, turn the
3687 // adjcallstackup instruction into a 'sub ESP, <amt>' and the
3688 // adjcallstackdown instruction into 'add ESP, <amt>'
3689
3690 // We need to keep the stack aligned properly. To do this, we round the
3691 // amount of space needed for the outgoing arguments up to the next
3692 // alignment boundary.
3693 Amount = alignTo(Amount, getStackAlign());
3694
3695 const Function &F = MF.getFunction();
3696 bool WindowsCFI = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
3697 bool DwarfCFI = !WindowsCFI && MF.needsFrameMoves();
3698
3699 // If we have any exception handlers in this function, and we adjust
3700 // the SP before calls, we may need to indicate this to the unwinder
3701 // using GNU_ARGS_SIZE. Note that this may be necessary even when
3702 // Amount == 0, because the preceding function may have set a non-0
3703 // GNU_ARGS_SIZE.
3704 // TODO: We don't need to reset this between subsequent functions,
3705 // if it didn't change.
3706 bool HasDwarfEHHandlers = !WindowsCFI && !MF.getLandingPads().empty();
3707
3708 if (HasDwarfEHHandlers && !isDestroy &&
3710 BuildCFI(MBB, InsertPos, DL,
3711 MCCFIInstruction::createGnuArgsSize(nullptr, Amount));
3712
3713 if (Amount == 0)
3714 return I;
3715
3716 // Factor out the amount that gets handled inside the sequence
3717 // (Pushes of argument for frame setup, callee pops for frame destroy)
3718 Amount -= InternalAmt;
3719
3720 // TODO: This is needed only if we require precise CFA.
3721 // If this is a callee-pop calling convention, emit a CFA adjust for
3722 // the amount the callee popped.
3723 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF))
3724 BuildCFI(MBB, InsertPos, DL,
3725 MCCFIInstruction::createAdjustCfaOffset(nullptr, -InternalAmt));
3726
3727 // Add Amount to SP to destroy a frame, or subtract to setup.
3728 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
3729
3730 if (StackAdjustment) {
3731 // Merge with any previous or following adjustment instruction. Note: the
3732 // instructions merged with here do not have CFI, so their stack
3733 // adjustments do not feed into CfaAdjustment.
3734 StackAdjustment += mergeSPUpdates(MBB, InsertPos, true);
3735 StackAdjustment += mergeSPUpdates(MBB, InsertPos, false);
3736
3737 if (StackAdjustment) {
3738 if (!(F.hasMinSize() &&
3739 adjustStackWithPops(MBB, InsertPos, DL, StackAdjustment)))
3740 BuildStackAdjustment(MBB, InsertPos, DL, StackAdjustment,
3741 /*InEpilogue=*/false);
3742 }
3743 }
3744
3745 if (DwarfCFI && !hasFP(MF)) {
3746 // If we don't have FP, but need to generate unwind information,
3747 // we need to set the correct CFA offset after the stack adjustment.
3748 // How much we adjust the CFA offset depends on whether we're emitting
3749 // CFI only for EH purposes or for debugging. EH only requires the CFA
3750 // offset to be correct at each call site, while for debugging we want
3751 // it to be more precise.
3752
3753 int64_t CfaAdjustment = -StackAdjustment;
3754 // TODO: When not using precise CFA, we also need to adjust for the
3755 // InternalAmt here.
3756 if (CfaAdjustment) {
3757 BuildCFI(
3758 MBB, InsertPos, DL,
3759 MCCFIInstruction::createAdjustCfaOffset(nullptr, CfaAdjustment));
3760 }
3761 }
3762
3763 return I;
3764 }
3765
3766 if (InternalAmt) {
3769 while (CI != B && !std::prev(CI)->isCall())
3770 --CI;
3771 BuildStackAdjustment(MBB, CI, DL, -InternalAmt, /*InEpilogue=*/false);
3772 }
3773
3774 return I;
3775}
3776
3778 assert(MBB.getParent() && "Block is not attached to a function!");
3779 const MachineFunction &MF = *MBB.getParent();
3780 if (!MBB.isLiveIn(X86::EFLAGS))
3781 return true;
3782
3783 // If stack probes have to loop inline or call, that will clobber EFLAGS.
3784 // FIXME: we could allow cases that will use emitStackProbeInlineGenericBlock.
3786 const X86TargetLowering &TLI = *STI.getTargetLowering();
3787 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
3788 return false;
3789
3791 return !TRI->hasStackRealignment(MF) && !X86FI->hasSwiftAsyncContext();
3792}
3793
3795 assert(MBB.getParent() && "Block is not attached to a function!");
3796
3797 // Win64 has strict requirements in terms of epilogue and we are
3798 // not taking a chance at messing with them.
3799 // I.e., unless this block is already an exit block, we can't use
3800 // it as an epilogue.
3801 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock())
3802 return false;
3803
3804 // Swift async context epilogue has a BTR instruction that clobbers parts of
3805 // EFLAGS.
3806 const MachineFunction &MF = *MBB.getParent();
3809
3811 return true;
3812
3813 // If we cannot use LEA to adjust SP, we may need to use ADD, which
3814 // clobbers the EFLAGS. Check that we do not need to preserve it,
3815 // otherwise, conservatively assume this is not
3816 // safe to insert the epilogue here.
3818}
3819
3821 // If we may need to emit frameless compact unwind information, give
3822 // up as this is currently broken: PR25614.
3823 bool CompactUnwind =
3825 nullptr;
3826 return (MF.getFunction().hasFnAttribute(Attribute::NoUnwind) || hasFP(MF) ||
3827 !CompactUnwind) &&
3828 // The lowering of segmented stack and HiPE only support entry
3829 // blocks as prologue blocks: PR26107. This limitation may be
3830 // lifted if we fix:
3831 // - adjustForSegmentedStacks
3832 // - adjustForHiPEPrologue
3834 !MF.shouldSplitStack();
3835}
3836
3839 const DebugLoc &DL, bool RestoreSP) const {
3840 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env");
3841 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32");
3842 assert(STI.is32Bit() && !Uses64BitFramePtr &&
3843 "restoring EBP/ESI on non-32-bit target");
3844
3845 MachineFunction &MF = *MBB.getParent();
3847 Register BasePtr = TRI->getBaseRegister();
3848 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo();
3850 MachineFrameInfo &MFI = MF.getFrameInfo();
3851
3852 // FIXME: Don't set FrameSetup flag in catchret case.
3853
3854 int FI = FuncInfo.EHRegNodeFrameIndex;
3855 int EHRegSize = MFI.getObjectSize(FI);
3856
3857 if (RestoreSP) {
3858 // MOV32rm -EHRegSize(%ebp), %esp
3859 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), X86::ESP),
3860 X86::EBP, true, -EHRegSize)
3862 }
3863
3864 Register UsedReg;
3865 int EHRegOffset = getFrameIndexReference(MF, FI, UsedReg).getFixed();
3866 int EndOffset = -EHRegOffset - EHRegSize;
3867 FuncInfo.EHRegNodeEndOffset = EndOffset;
3868
3869 if (UsedReg == FramePtr) {
3870 // ADD $offset, %ebp
3871 unsigned ADDri = getADDriOpcode(false);
3872 BuildMI(MBB, MBBI, DL, TII.get(ADDri), FramePtr)
3874 .addImm(EndOffset)
3876 ->getOperand(3)
3877 .setIsDead();
3878 assert(EndOffset >= 0 &&
3879 "end of registration object above normal EBP position!");
3880 } else if (UsedReg == BasePtr) {
3881 // LEA offset(%ebp), %esi
3882 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA32r), BasePtr),
3883 FramePtr, false, EndOffset)
3885 // MOV32rm SavedEBPOffset(%esi), %ebp
3886 assert(X86FI->getHasSEHFramePtrSave());
3887 int Offset =
3888 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg)
3889 .getFixed();
3890 assert(UsedReg == BasePtr);
3891 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), FramePtr),
3892 UsedReg, true, Offset)
3894 } else {
3895 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr");
3896 }
3897 return MBBI;
3898}
3899
3901 return TRI->getSlotSize();
3902}
3903
3906 return StackPtr;
3907}
3908
3912 Register FrameRegister = RI->getFrameRegister(MF);
3913 if (getInitialCFARegister(MF) == FrameRegister &&
3915 DwarfFrameBase FrameBase;
3916 FrameBase.Kind = DwarfFrameBase::CFA;
3917 FrameBase.Location.Offset =
3919 return FrameBase;
3920 }
3921
3922 return DwarfFrameBase{DwarfFrameBase::Register, {FrameRegister}};
3923}
3924
3925namespace {
3926// Struct used by orderFrameObjects to help sort the stack objects.
3927struct X86FrameSortingObject {
3928 bool IsValid = false; // true if we care about this Object.
3929 unsigned ObjectIndex = 0; // Index of Object into MFI list.
3930 unsigned ObjectSize = 0; // Size of Object in bytes.
3931 Align ObjectAlignment = Align(1); // Alignment of Object in bytes.
3932 unsigned ObjectNumUses = 0; // Object static number of uses.
3933};
3934
3935// The comparison function we use for std::sort to order our local
3936// stack symbols. The current algorithm is to use an estimated
3937// "density". This takes into consideration the size and number of
3938// uses each object has in order to roughly minimize code size.
3939// So, for example, an object of size 16B that is referenced 5 times
3940// will get higher priority than 4 4B objects referenced 1 time each.
3941// It's not perfect and we may be able to squeeze a few more bytes out of
3942// it (for example : 0(esp) requires fewer bytes, symbols allocated at the
3943// fringe end can have special consideration, given their size is less
3944// important, etc.), but the algorithmic complexity grows too much to be
3945// worth the extra gains we get. This gets us pretty close.
3946// The final order leaves us with objects with highest priority going
3947// at the end of our list.
3948struct X86FrameSortingComparator {
3949 inline bool operator()(const X86FrameSortingObject &A,
3950 const X86FrameSortingObject &B) const {
3951 uint64_t DensityAScaled, DensityBScaled;
3952
3953 // For consistency in our comparison, all invalid objects are placed
3954 // at the end. This also allows us to stop walking when we hit the
3955 // first invalid item after it's all sorted.
3956 if (!A.IsValid)
3957 return false;
3958 if (!B.IsValid)
3959 return true;
3960
3961 // The density is calculated by doing :
3962 // (double)DensityA = A.ObjectNumUses / A.ObjectSize
3963 // (double)DensityB = B.ObjectNumUses / B.ObjectSize
3964 // Since this approach may cause inconsistencies in
3965 // the floating point <, >, == comparisons, depending on the floating
3966 // point model with which the compiler was built, we're going
3967 // to scale both sides by multiplying with
3968 // A.ObjectSize * B.ObjectSize. This ends up factoring away
3969 // the division and, with it, the need for any floating point
3970 // arithmetic.
3971 DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) *
3972 static_cast<uint64_t>(B.ObjectSize);
3973 DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) *
3974 static_cast<uint64_t>(A.ObjectSize);
3975
3976 // If the two densities are equal, prioritize highest alignment
3977 // objects. This allows for similar alignment objects
3978 // to be packed together (given the same density).
3979 // There's room for improvement here, also, since we can pack
3980 // similar alignment (different density) objects next to each
3981 // other to save padding. This will also require further
3982 // complexity/iterations, and the overall gain isn't worth it,
3983 // in general. Something to keep in mind, though.
3984 if (DensityAScaled == DensityBScaled)
3985 return A.ObjectAlignment < B.ObjectAlignment;
3986
3987 return DensityAScaled < DensityBScaled;
3988 }
3989};
3990} // namespace
3991
3992// Order the symbols in the local stack.
3993// We want to place the local stack objects in some sort of sensible order.
3994// The heuristic we use is to try and pack them according to static number
3995// of uses and size of object in order to minimize code size.
3997 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
3998 const MachineFrameInfo &MFI = MF.getFrameInfo();
3999
4000 // Don't waste time if there's nothing to do.
4001 if (ObjectsToAllocate.empty())
4002 return;
4003
4004 // Create an array of all MFI objects. We won't need all of these
4005 // objects, but we're going to create a full array of them to make
4006 // it easier to index into when we're counting "uses" down below.
4007 // We want to be able to easily/cheaply access an object by simply
4008 // indexing into it, instead of having to search for it every time.
4009 std::vector<X86FrameSortingObject> SortingObjects(MFI.getObjectIndexEnd());
4010
4011 // Walk the objects we care about and mark them as such in our working
4012 // struct.
4013 for (auto &Obj : ObjectsToAllocate) {
4014 SortingObjects[Obj].IsValid = true;
4015 SortingObjects[Obj].ObjectIndex = Obj;
4016 SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlign(Obj);
4017 // Set the size.
4018 int ObjectSize = MFI.getObjectSize(Obj);
4019 if (ObjectSize == 0)
4020 // Variable size. Just use 4.
4021 SortingObjects[Obj].ObjectSize = 4;
4022 else
4023 SortingObjects[Obj].ObjectSize = ObjectSize;
4024 }
4025
4026 // Count the number of uses for each object.
4027 for (auto &MBB : MF) {
4028 for (auto &MI : MBB) {
4029 if (MI.isDebugInstr())
4030 continue;
4031 for (const MachineOperand &MO : MI.operands()) {
4032 // Check to see if it's a local stack symbol.
4033 if (!MO.isFI())
4034 continue;
4035 int Index = MO.getIndex();
4036 // Check to see if it falls within our range, and is tagged
4037 // to require ordering.
4038 if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
4039 SortingObjects[Index].IsValid)
4040 SortingObjects[Index].ObjectNumUses++;
4041 }
4042 }
4043 }
4044
4045 // Sort the objects using X86FrameSortingAlgorithm (see its comment for
4046 // info).
4047 llvm::stable_sort(SortingObjects, X86FrameSortingComparator());
4048
4049 // Now modify the original list to represent the final order that
4050 // we want. The order will depend on whether we're going to access them
4051 // from the stack pointer or the frame pointer. For SP, the list should
4052 // end up with the END containing objects that we want with smaller offsets.
4053 // For FP, it should be flipped.
4054 int i = 0;
4055 for (auto &Obj : SortingObjects) {
4056 // All invalid items are sorted at the end, so it's safe to stop.
4057 if (!Obj.IsValid)
4058 break;
4059 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4060 }
4061
4062 // Flip it if we're accessing off of the FP.
4063 if (!TRI->hasStackRealignment(MF) && hasFP(MF))
4064 std::reverse(ObjectsToAllocate.begin(), ObjectsToAllocate.end());
4065}
4066
4067unsigned
4069 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue.
4070 unsigned Offset = 16;
4071 // RBP is immediately pushed.
4072 Offset += SlotSize;
4073 // All callee-saved registers are then pushed.
4074 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
4075 // Every funclet allocates enough stack space for the largest outgoing call.
4076 Offset += getWinEHFuncletFrameSize(MF);
4077 return Offset;
4078}
4079
4081 MachineFunction &MF, RegScavenger *RS) const {
4082 // Mark the function as not having WinCFI. We will set it back to true in
4083 // emitPrologue if it gets called and emits CFI.
4084 MF.setHasWinCFI(false);
4085
4086 // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
4087 // aligned. The format doesn't support misaligned stack adjustments.
4090
4091 // If this function isn't doing Win64-style C++ EH, we don't need to do
4092 // anything.
4093 if (STI.is64Bit() && MF.hasEHFunclets() &&
4096 adjustFrameForMsvcCxxEh(MF);
4097 }
4098}
4099
4100void X86FrameLowering::adjustFrameForMsvcCxxEh(MachineFunction &MF) const {
4101 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset
4102 // relative to RSP after the prologue. Find the offset of the last fixed
4103 // object, so that we can allocate a slot immediately following it. If there
4104 // were no fixed objects, use offset -SlotSize, which is immediately after the
4105 // return address. Fixed objects have negative frame indices.
4106 MachineFrameInfo &MFI = MF.getFrameInfo();
4107 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
4108 int64_t MinFixedObjOffset = -SlotSize;
4109 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I)
4110 MinFixedObjOffset = std::min(MinFixedObjOffset, MFI.getObjectOffset(I));
4111
4112 for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
4113 for (WinEHHandlerType &H : TBME.HandlerArray) {
4114 int FrameIndex = H.CatchObj.FrameIndex;
4115 if (FrameIndex != INT_MAX) {
4116 // Ensure alignment.
4117 unsigned Align = MFI.getObjectAlign(FrameIndex).value();
4118 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % Align;
4119 MinFixedObjOffset -= MFI.getObjectSize(FrameIndex);
4120 MFI.setObjectOffset(FrameIndex, MinFixedObjOffset);
4121 }
4122 }
4123 }
4124
4125 // Ensure alignment.
4126 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8;
4127 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize;
4128 int UnwindHelpFI =
4129 MFI.CreateFixedObject(SlotSize, UnwindHelpOffset, /*IsImmutable=*/false);
4130 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
4131
4132 // Store -2 into UnwindHelp on function entry. We have to scan forwards past
4133 // other frame setup instructions.
4134 MachineBasicBlock &MBB = MF.front();
4135 auto MBBI = MBB.begin();
4136 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
4137 ++MBBI;
4138
4140 addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mi32)),
4141 UnwindHelpFI)
4142 .addImm(-2);
4143}
4144
4146 MachineFunction &MF, RegScavenger *RS) const {
4147 auto *X86FI = MF.getInfo<X86MachineFunctionInfo>();
4148
4149 if (STI.is32Bit() && MF.hasEHFunclets())
4151 // We have emitted prolog and epilog. Don't need stack pointer saving
4152 // instruction any more.
4153 if (MachineInstr *MI = X86FI->getStackPtrSaveMI()) {
4154 MI->eraseFromParent();
4155 X86FI->setStackPtrSaveMI(nullptr);
4156 }
4157}
4158
4160 MachineFunction &MF) const {
4161 // 32-bit functions have to restore stack pointers when control is transferred
4162 // back to the parent function. These blocks are identified as eh pads that
4163 // are not funclet entries.
4164 bool IsSEH = isAsynchronousEHPersonality(
4166 for (MachineBasicBlock &MBB : MF) {
4167 bool NeedsRestore = MBB.isEHPad() && !MBB.isEHFuncletEntry();
4168 if (NeedsRestore)
4170 /*RestoreSP=*/IsSEH);
4171 }
4172}
unsigned const MachineRegisterInfo * MRI
static bool isFuncletReturnInstr(const MachineInstr &MI)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const uint64_t kSplitStackAvailable
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Given that RA is a live value
uint64_t Size
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static bool isTailCallOpcode(unsigned Opc)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define H(x, y, z)
Definition: MD5.cpp:57
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
static bool is64Bit(const char *name)
static unsigned calculateSetFPREG(uint64_t SPAdjust)
static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary)
GetScratchRegister - Get a temp register for performing work in the segmented stack and the Erlang/Hi...
static unsigned getADDriOpcode(bool IsLP64)
static unsigned getPUSH2Opcode(const X86Subtarget &ST)
static unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm)
static unsigned getLEArOpcode(bool IsLP64)
static unsigned getSUBriOpcode(bool IsLP64)
static bool flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB)
Check if the flags need to be preserved before the terminators.
static unsigned getANDriOpcode(bool IsLP64, int64_t Imm)
static bool isEAXLiveIn(MachineBasicBlock &MBB)
static unsigned getADDrrOpcode(bool IsLP64)
static bool HasNestArgument(const MachineFunction *MF)
static unsigned getPOPOpcode(const X86Subtarget &ST)
static unsigned getPOP2Opcode(const X86Subtarget &ST)
static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD, const StringRef LiteralName)
Lookup an ERTS parameter in the !hipe.literals named metadata node.
static bool blockEndIsUnreachable(const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI)
static unsigned getSUBrrOpcode(bool IsLP64)
static unsigned getPUSHOpcode(const X86Subtarget &ST)
static const unsigned FramePtr
static constexpr uint32_t RegMask
Definition: aarch32.h:221
static constexpr uint32_t Opcode
Definition: aarch32.h:200
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
reverse_iterator rend() const
Definition: ArrayRef.h:157
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
reverse_iterator rbegin() const
Definition: ArrayRef.h:156
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
BitVector & reset()
Definition: BitVector.h:392
BitVector & set()
Definition: BitVector.h:351
iterator_range< const_set_bits_iterator > set_bits() const
Definition: