39#define DEBUG_TYPE "x86-fl"
41STATISTIC(NumFrameLoopProbe,
"Number of loop stack probes used in prologue");
43 "Number of extra stack probes generated in prologue");
44STATISTIC(NumFunctionUsingPush2Pop2,
"Number of funtions using push2/pop2");
52 STI(STI),
TII(*STI.getInstrInfo()),
TRI(STI.getRegisterInfo()) {
76 (
hasFP(MF) && !
TRI->hasStackRealignment(MF)) ||
109 return IsLP64 ? X86::SUB64ri32 : X86::SUB32ri;
113 return IsLP64 ? X86::ADD64ri32 : X86::ADD32ri;
117 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
121 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
125 return IsLP64 ? X86::AND64ri32 : X86::AND32ri;
129 return IsLP64 ? X86::LEA64r : X86::LEA32r;
135 return X86::MOV32ri64;
137 return X86::MOV64ri32;
161 return ST.is64Bit() ? (ST.hasPPX() ? X86::PUSHP64r : X86::PUSH64r)
165 return ST.is64Bit() ? (ST.hasPPX() ? X86::POPP64r : X86::POP64r)
169 return ST.hasPPX() ? X86::PUSH2P : X86::PUSH2;
172 return ST.hasPPX() ? X86::POP2P : X86::POP2;
177 unsigned Reg =
RegMask.PhysReg;
179 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
180 Reg == X86::AH || Reg == X86::AL)
194 bool BreakNext =
false;
199 if (Reg != X86::EFLAGS)
220 if (Succ->isLiveIn(X86::EFLAGS))
231 bool InEpilogue)
const {
232 bool isSub = NumBytes < 0;
246 if (EmitInlineStackProbe && !InEpilogue) {
252 }
else if (
Offset > Chunk) {
263 unsigned AddSubRROpc =
272 MI->getOperand(3).setIsDead();
274 }
else if (
Offset > 8 * Chunk) {
298 MI->getOperand(3).setIsDead();
318 unsigned Opc = isSub ? (
Is64Bit ? X86::PUSH64r : X86::PUSH32r)
319 : (
Is64Bit ? X86::POP64r : X86::POP32r);
328 BuildStackAdjustment(
MBB,
MBBI,
DL, isSub ? -ThisVal : ThisVal, InEpilogue)
338 assert(
Offset != 0 &&
"zero offset stack adjustment requested");
356 if (UseLEA && !
STI.useLeaForSP())
361 "We shouldn't have allowed this insertion point");
378 MI->getOperand(3).setIsDead();
385 bool doMergeWithPrevious)
const {
404 if (doMergeWithPrevious && PI !=
MBB.
begin() && PI->isCFIInstruction())
407 unsigned Opc = PI->getOpcode();
410 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri) &&
411 PI->getOperand(0).getReg() ==
StackPtr) {
413 Offset = PI->getOperand(2).getImm();
414 }
else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
415 PI->getOperand(0).getReg() ==
StackPtr &&
416 PI->getOperand(1).getReg() ==
StackPtr &&
417 PI->getOperand(2).getImm() == 1 &&
418 PI->getOperand(3).getReg() == X86::NoRegister &&
419 PI->getOperand(5).getReg() == X86::NoRegister) {
421 Offset = PI->getOperand(4).getImm();
422 }
else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri) &&
423 PI->getOperand(0).getReg() ==
StackPtr) {
425 Offset = -PI->getOperand(2).getImm();
430 if (PI !=
MBB.
end() && PI->isCFIInstruction()) {
437 if (!doMergeWithPrevious)
474 unsigned DwarfReg =
MRI->getDwarfRegNum(MachineFramePtr,
true);
498 unsigned DwarfReg =
MRI->getDwarfRegNum(Reg,
true);
509 CfaExpr.
push_back(dwarf::DW_CFA_expression);
518 unsigned DwarfFramePtr =
MRI->getDwarfRegNum(MachineFramePtr,
true);
519 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
534 int FI =
MI->getOperand(1).getIndex();
542 unsigned DwarfFramePtr =
MRI->getDwarfRegNum(MachineFramePtr,
true);
543 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
549 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
559void X86FrameLowering::emitZeroCallUsedRegs(
BitVector RegsToZero,
575 if (!X86::RFP80RegClass.
contains(Reg))
578 unsigned NumFPRegs = ST.is64Bit() ? 8 : 7;
579 for (
unsigned i = 0; i != NumFPRegs; ++i)
582 for (
unsigned i = 0; i != NumFPRegs; ++i)
590 if (
TRI->isGeneralPurposeRegister(MF, Reg)) {
592 RegsToZero.
reset(Reg);
607 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum)
const {
614 emitStackProbeInline(MF,
MBB,
MBBI,
DL,
false);
617 emitStackProbeCall(MF,
MBB,
MBBI,
DL, InProlog, InstrNum);
628 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
630 if (Where != PrologMBB.
end()) {
632 emitStackProbeInline(MF, PrologMBB, Where,
DL,
true);
633 Where->eraseFromParent();
641 bool InProlog)
const {
644 emitStackProbeInlineWindowsCoreCLR64(MF,
MBB,
MBBI,
DL, InProlog);
646 emitStackProbeInlineGeneric(MF,
MBB,
MBBI,
DL, InProlog);
649void X86FrameLowering::emitStackProbeInlineGeneric(
658 "different expansion expected for CoreCLR 64 bit");
660 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
661 uint64_t ProbeChunk = StackProbeSize * 8;
664 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
669 if (
Offset > ProbeChunk) {
671 MaxAlign % StackProbeSize);
674 MaxAlign % StackProbeSize);
678void X86FrameLowering::emitStackProbeInlineGenericBlock(
683 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
687 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
688 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
692 assert(AlignOffset < StackProbeSize);
695 if (StackProbeSize <
Offset + AlignOffset) {
698 BuildStackAdjustment(
MBB,
MBBI,
DL, -StackAdjustment,
false)
700 if (!HasFP && NeedsDwarfCFI) {
711 NumFrameExtraProbe++;
712 CurrentOffset = StackProbeSize - AlignOffset;
718 while (CurrentOffset + StackProbeSize <
Offset) {
719 BuildStackAdjustment(
MBB,
MBBI,
DL, -StackProbeSize,
false)
722 if (!HasFP && NeedsDwarfCFI) {
732 NumFrameExtraProbe++;
733 CurrentOffset += StackProbeSize;
742 unsigned Opc =
Is64Bit ? X86::PUSH64r : X86::PUSH32r;
747 BuildStackAdjustment(
MBB,
MBBI,
DL, -ChunkSize,
false)
754void X86FrameLowering::emitStackProbeInlineGenericLoop(
762 "Inline stack probe loop will clobber live EFLAGS.");
764 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
768 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
769 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
772 if (AlignOffset < StackProbeSize) {
774 BuildStackAdjustment(
MBB,
MBBI,
DL, -AlignOffset,
false)
782 NumFrameExtraProbe++;
795 MF.
insert(MBBIter, testMBB);
796 MF.
insert(MBBIter, tailMBB);
817 if (!HasFP && NeedsDwarfCFI) {
820 const Register DwarfFinalStackProbed =
827 nullptr,
TRI->getDwarfRegNum(DwarfFinalStackProbed,
true)));
834 BuildStackAdjustment(*testMBB, testMBB->
end(),
DL, -StackProbeSize,
868 BuildStackAdjustment(*tailMBB, TailMBBIter,
DL, -TailOffset,
874 if (!HasFP && NeedsDwarfCFI) {
884 nullptr,
TRI->getDwarfRegNum(DwarfStackPtr,
true)));
892void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
896 assert(
STI.is64Bit() &&
"different expansion needed for 32 bit");
903 "Inline stack probe loop will clobber live EFLAGS.");
938 MF.
insert(MBBIter, RoundMBB);
939 MF.
insert(MBBIter, LoopMBB);
940 MF.
insert(MBBIter, ContinueMBB);
948 const int64_t ThreadEnvironmentStackLimit = 0x10;
950 const int64_t PageMask = ~(
PageSize - 1);
957 SizeReg = InProlog ? X86::RAX :
MRI.createVirtualRegister(RegClass),
958 ZeroReg = InProlog ? X86::RCX :
MRI.createVirtualRegister(RegClass),
959 CopyReg = InProlog ? X86::RDX :
MRI.createVirtualRegister(RegClass),
960 TestReg = InProlog ? X86::RDX :
MRI.createVirtualRegister(RegClass),
961 FinalReg = InProlog ? X86::RDX :
MRI.createVirtualRegister(RegClass),
962 RoundedReg = InProlog ? X86::RDX :
MRI.createVirtualRegister(RegClass),
963 LimitReg = InProlog ? X86::RCX :
MRI.createVirtualRegister(RegClass),
964 JoinReg = InProlog ? X86::RCX :
MRI.createVirtualRegister(RegClass),
965 ProbeReg = InProlog ? X86::RCX :
MRI.createVirtualRegister(RegClass);
968 int64_t RCXShadowSlot = 0;
969 int64_t RDXShadowSlot = 0;
985 int64_t InitSlot = 8 + CalleeSaveSize + (
HasFP ? 8 : 0);
989 RCXShadowSlot = InitSlot;
991 RDXShadowSlot = InitSlot;
992 if (IsRDXLiveIn && IsRCXLiveIn)
1033 .
addImm(ThreadEnvironmentStackLimit)
1043 BuildMI(RoundMBB,
DL,
TII.get(X86::AND64ri32), RoundedReg)
1086 TII.get(X86::MOV64rm), X86::RCX),
1087 X86::RSP,
false, RCXShadowSlot);
1090 TII.get(X86::MOV64rm), X86::RDX),
1091 X86::RSP,
false, RDXShadowSlot);
1097 BuildMI(*ContinueMBB, ContinueMBBI,
DL,
TII.get(X86::SUB64rr), X86::RSP)
1110 for (++BeforeMBBI; BeforeMBBI !=
MBB.
end(); ++BeforeMBBI) {
1126void X86FrameLowering::emitStackProbeCall(
1129 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum)
const {
1135 "code model and indirect thunks not yet implemented.");
1139 "Stack probe calls will clobber live EFLAGS.");
1143 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1145 CallOp = X86::CALLpcrel32;
1206 for (++ExpansionMBBI; ExpansionMBBI !=
MBBI; ++ExpansionMBBI)
1214 const uint64_t Win64MaxSEHOffset = 128;
1215 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
1217 return SEHFrameOffset & -16;
1225X86FrameLowering::calculateMaxStackAlign(
const MachineFunction &MF)
const {
1232 MaxAlign = (
StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1239 MaxAlign = (MaxAlign > 16) ? MaxAlign :
Align(16);
1241 MaxAlign =
Align(16);
1243 return MaxAlign.
value();
1256 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1257 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1262 if (Reg ==
StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1264 NumFrameLoopProbe++;
1275 MF.
insert(MBBIter, entryMBB);
1276 MF.
insert(MBBIter, headMBB);
1277 MF.
insert(MBBIter, bodyMBB);
1278 MF.
insert(MBBIter, footMBB);
1279 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1288 BuildMI(entryMBB,
DL,
TII.get(TargetOpcode::COPY), FinalStackProbed)
1292 BuildMI(entryMBB,
DL,
TII.get(AndOp), FinalStackProbed)
1293 .
addReg(FinalStackProbed)
1298 MI->getOperand(3).setIsDead();
1302 .
addReg(FinalStackProbed)
1325 .
addReg(FinalStackProbed)
1355 .
addReg(FinalStackProbed)
1371 .
addReg(FinalStackProbed)
1393 MI->getOperand(3).setIsDead();
1401 "MF used frame lowering for wrong subtarget");
1410bool X86FrameLowering::isWin64Prologue(
const MachineFunction &MF)
const {
1414bool X86FrameLowering::needsDwarfCFI(
const MachineFunction &MF)
const {
1506 "MF used frame lowering for wrong subtarget");
1512 uint64_t MaxAlign = calculateMaxStackAlign(MF);
1518 bool FnHasClrFunclet =
1520 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1521 bool HasFP =
hasFP(MF);
1522 bool IsWin64Prologue = isWin64Prologue(MF);
1527 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1528 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1534 bool HasWinCFI =
false;
1545 ArgBaseReg =
MI->getOperand(0).getReg();
1558 if (NeedsDwarfCFI) {
1560 unsigned DwarfStackPtr =
TRI->getDwarfRegNum(ArgBaseReg,
true);
1579 if (TailCallArgReserveSize && IsWin64Prologue)
1582 const bool EmitStackProbeCall =
1599 .
addUse(X86::NoRegister);
1646 !EmitStackProbeCall &&
1654 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
1661 if (TailCallArgReserveSize != 0) {
1662 BuildStackAdjustment(
MBB,
MBBI,
DL, -(
int)TailCallArgReserveSize,
1685 Register Establisher = X86::NoRegister;
1691 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1711 if (
TRI->hasStackRealignment(MF) && !IsWin64Prologue)
1712 NumBytes =
alignTo(NumBytes, MaxAlign);
1720 if (NeedsDwarfCFI && !ArgBaseReg.
isValid()) {
1726 nullptr, -2 * stackGrowth + (
int)TailCallArgReserveSize),
1730 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1734 (
int)TailCallArgReserveSize),
1752 if (Attrs.hasAttrSomewhere(Attribute::SwiftAsync)) {
1787 if (!IsWin64Prologue && !IsFunclet) {
1796 if (NeedsDwarfCFI) {
1799 CfaExpr.
push_back(dwarf::DW_CFA_expression);
1801 unsigned DwarfReg =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1804 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
1813 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1832 assert(!IsFunclet &&
"funclets without FPs not yet implemented");
1840 if (HasFP &&
TRI->hasStackRealignment(MF))
1848 unsigned ParentFrameNumBytes = NumBytes;
1850 NumBytes = getWinEHFuncletFrameSize(MF);
1853 bool PushedRegs =
false;
1859 unsigned Opc =
MBBI->getOpcode();
1860 return Opc == X86::PUSH32r || Opc == X86::PUSH64r || Opc == X86::PUSHP64r ||
1861 Opc == X86::PUSH2 || Opc == X86::PUSH2P;
1864 while (IsCSPush(
MBBI)) {
1869 unsigned Opc = LastCSPush->getOpcode();
1871 if (!HasFP && NeedsDwarfCFI) {
1877 if (Opc == X86::PUSH2 || Opc == X86::PUSH2P)
1890 if (Opc == X86::PUSH2 || Opc == X86::PUSH2P)
1892 .
addImm(LastCSPush->getOperand(1).getReg())
1900 if (!IsWin64Prologue && !IsFunclet &&
TRI->hasStackRealignment(MF) &&
1902 assert(HasFP &&
"There should be a frame pointer if stack is realigned.");
1928 uint64_t AlignedNumBytes = NumBytes;
1929 if (IsWin64Prologue && !IsFunclet &&
TRI->hasStackRealignment(MF))
1930 AlignedNumBytes =
alignTo(AlignedNumBytes, MaxAlign);
1931 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
1933 "The Red Zone is not accounted for in stack probes");
1955 int64_t
Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
1963 .
addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
1982 }
else if (NumBytes) {
1986 if (NeedsWinCFI && NumBytes) {
1993 int SEHFrameOffset = 0;
1994 unsigned SPOrEstablisher;
2001 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2005 Establisher,
false, PSPSlotOffset)
2012 false, PSPSlotOffset)
2019 SPOrEstablisher = Establisher;
2024 if (IsWin64Prologue && HasFP) {
2031 SPOrEstablisher,
false, SEHFrameOffset);
2034 .
addReg(SPOrEstablisher);
2037 if (NeedsWinCFI && !IsFunclet) {
2038 assert(!NeedsWinFPO &&
"this setframe incompatible with FPO data");
2047 }
else if (IsFunclet &&
STI.is32Bit()) {
2071 if (X86::FR64RegClass.
contains(Reg)) {
2074 if (IsWin64Prologue && IsFunclet)
2082 assert(!NeedsWinFPO &&
"SEH_SaveXMM incompatible with FPO data");
2092 if (NeedsWinCFI && HasWinCFI)
2096 if (FnHasClrFunclet && !IsFunclet) {
2100 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2114 if (IsWin64Prologue &&
TRI->hasStackRealignment(MF)) {
2115 assert(HasFP &&
"There should be a frame pointer if stack is realigned.");
2116 BuildStackAlignAND(
MBB,
MBBI,
DL, SPOrEstablisher, MaxAlign);
2120 if (IsFunclet &&
STI.is32Bit())
2153 assert(UsedReg == BasePtr);
2162 int FI =
MI->getOperand(1).getIndex();
2163 unsigned MOVmr =
Is64Bit ? X86::MOV64mr : X86::MOV32mr;
2170 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2172 if (!HasFP && NumBytes) {
2213 switch (
MI.getOpcode()) {
2215 case X86::CLEANUPRET:
2237X86FrameLowering::getPSPSlotOffsetFromSP(
const MachineFunction &MF)
const {
2244 return static_cast<unsigned>(
Offset);
2248X86FrameLowering::getWinEHFuncletFrameSize(
const MachineFunction &MF)
const {
2255 WinEHXMMSlotInfo.
size() *
TRI->getSpillSize(X86::VR128RegClass);
2264 UsedSize = getPSPSlotOffsetFromSP(MF) +
SlotSize;
2275 return FrameSizeMinusRBP + XMMSize - CSSize;
2279 return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi ||
2280 Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
2281 Opc == X86::TCRETURNdi64 || Opc == X86::TCRETURNmi64;
2292 DL =
MBBI->getDebugLoc();
2300 bool NeedsWin64CFI =
2306 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2309 bool HasFP =
hasFP(MF);
2318 unsigned Opc = X86::LEA32r;
2320 ArgBaseReg =
MI->getOperand(0).getReg();
2321 if (
STI.is64Bit()) {
2323 StackReg = X86::RSP;
2334 if (NeedsDwarfCFI) {
2335 unsigned DwarfStackPtr =
TRI->getDwarfRegNum(StackReg,
true);
2345 assert(HasFP &&
"EH funclets without FP not yet implemented");
2346 NumBytes = getWinEHFuncletFrameSize(MF);
2350 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2354 if (
TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2355 NumBytes =
alignTo(FrameSize, MaxAlign);
2357 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2359 uint64_t SEHStackAllocAmt = NumBytes;
2384 if (NeedsDwarfCFI) {
2386 unsigned DwarfStackPtr =
2387 TRI->getDwarfRegNum(
Is64Bit ? X86::RSP : X86::ESP,
true);
2393 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
2408 unsigned Opc = PI->getOpcode();
2410 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2412 (Opc != X86::POP32r && Opc != X86::POP64r && Opc != X86::BTR64ri8 &&
2413 Opc != X86::ADD64ri32 && Opc != X86::POPP64r && Opc != X86::POP2 &&
2414 Opc != X86::POP2P && Opc != X86::LEA64r))
2424 int FI =
MI->getOperand(1).getIndex();
2425 unsigned MOVrm =
Is64Bit ? X86::MOV64rm : X86::MOV32rm;
2432 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2433 emitCatchRetReturnValue(
MBB, FirstCSPop, &*Terminator);
2436 DL =
MBBI->getDebugLoc();
2448 if (
TRI->hasStackRealignment(MF))
2452 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2464 if (LEAAmount != 0) {
2474 }
else if (NumBytes) {
2477 if (!HasFP && NeedsDwarfCFI) {
2481 nullptr, CSSize + TailCallArgReserveSize +
SlotSize),
2496 if (!HasFP && NeedsDwarfCFI) {
2503 unsigned Opc = PI->getOpcode();
2505 if (Opc == X86::POP32r || Opc == X86::POP64r || Opc == X86::POPP64r ||
2506 Opc == X86::POP2 || Opc == X86::POP2P) {
2510 if (Opc == X86::POP2 || Opc == X86::POP2P)
2528 assert(
Offset >= 0 &&
"TCDelta should never be positive");
2552 else if (
TRI->hasStackRealignment(MF))
2566 int64_t FPDelta = 0;
2577 if (IsWin64Prologue) {
2586 uint64_t NumBytes = FrameSize - CSSize;
2596 FPDelta = FrameSize - SEHFrameOffset;
2598 "FPDelta isn't aligned per the Win64 ABI!");
2610 if (TailCallReturnAddrDelta < 0)
2611 Offset -= TailCallReturnAddrDelta;
2629 const auto it = WinEHXMMSlotInfo.find(FI);
2631 if (it == WinEHXMMSlotInfo.end())
2642 int Adjustment)
const {
2652 bool IgnoreSPUpdates)
const {
2702 "we don't handle this case!");
2734 std::vector<CalleeSavedInfo> &CSI)
const {
2738 unsigned CalleeSavedFrameSize = 0;
2739 unsigned XMMCalleeSavedFrameSize = 0;
2745 if (TailCallReturnAddrDelta < 0) {
2756 TailCallReturnAddrDelta -
SlotSize,
true);
2760 if (this->TRI->hasBasePointer(MF)) {
2786 for (
unsigned i = 0; i < CSI.size(); ++i) {
2787 if (
TRI->regsOverlap(CSI[i].getReg(), FPReg)) {
2788 CSI.erase(CSI.begin() + i);
2803 unsigned NumRegsForPush2 = 0;
2804 if (
STI.hasPush2Pop2()) {
2806 return X86::GR64RegClass.contains(
I.getReg());
2808 bool NeedPadding = (SpillSlotOffset % 16 != 0) && (NumCSGPR % 2 == 0);
2809 bool UsePush2Pop2 = NeedPadding ? NumCSGPR > 2 : NumCSGPR > 1;
2811 NumRegsForPush2 = UsePush2Pop2 ?
alignDown(NumCSGPR, 2) : 0;
2828 (SpillSlotOffset % 16 == 0 ||
2850 "Expect even candidates for push2/pop2");
2852 ++NumFunctionUsingPush2Pop2;
2863 MVT VT = MVT::Other;
2864 if (X86::VK16RegClass.
contains(Reg))
2865 VT =
STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2868 unsigned Size =
TRI->getSpillSize(*RC);
2869 Align Alignment =
TRI->getSpillAlign(*RC);
2871 assert(SpillSlotOffset < 0 &&
"SpillSlotOffset should always < 0 on X86");
2872 SpillSlotOffset = -
alignTo(-SpillSlotOffset, Alignment);
2875 SpillSlotOffset -=
Size;
2881 if (X86::VR128RegClass.
contains(Reg)) {
2882 WinEHXMMSlotInfo[
SlotIndex] = XMMCalleeSavedFrameSize;
2883 XMMCalleeSavedFrameSize +=
Size;
2908 auto UpdateLiveInCheckCanKill = [&](
Register Reg) {
2915 if (
MRI.isLiveIn(Reg))
2920 if (
MRI.isLiveIn(*AReg))
2924 auto UpdateLiveInGetKillRegState = [&](
Register Reg) {
2928 for (
auto RI = CSI.
rbegin(), RE = CSI.
rend(); RI != RE; ++RI) {
2936 .
addReg(Reg, UpdateLiveInGetKillRegState(Reg))
2937 .
addReg(Reg2, UpdateLiveInGetKillRegState(Reg2))
2941 .
addReg(Reg, UpdateLiveInGetKillRegState(Reg))
2947 unsigned Opc =
STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2948 Register BaseReg = this->TRI->getBaseRegister();
2962 MVT VT = MVT::Other;
2963 if (X86::VK16RegClass.
contains(Reg))
2964 VT =
STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2986 "SEH should not use CATCHRET");
2991 if (
STI.is64Bit()) {
3023 if (
MI->getOpcode() == X86::CATCHRET) {
3041 MVT VT = MVT::Other;
3042 if (X86::VK16RegClass.
contains(Reg))
3043 VT =
STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
3054 unsigned Opc =
STI.is64Bit() ? X86::POP64r : X86::POP32r;
3055 Register BaseReg = this->TRI->getBaseRegister();
3090 SavedRegs.
set(BasePtr);
3098 if (
I->hasNestAttr() && !
I->use_empty())
3115 return Primary ? X86::R14 : X86::R13;
3117 return Primary ? X86::EBX : X86::EDI;
3122 return Primary ? X86::R11 : X86::R12;
3124 return Primary ? X86::R11D : X86::R12D;
3134 "nested function.");
3135 return Primary ? X86::EAX : X86::ECX;
3138 return Primary ? X86::EDX : X86::EAX;
3139 return Primary ? X86::ECX : X86::EAX;
3150 unsigned TlsReg, TlsOffset;
3155 assert(&(*MF.
begin()) == &PrologueMBB &&
"Shrink-wrapping not supported yet");
3159 "Scratch register is live-in");
3179 bool IsNested =
false;
3188 for (
const auto &LI : PrologueMBB.
liveins()) {
3207 TlsOffset =
IsLP64 ? 0x70 : 0x40;
3210 TlsOffset = 0x60 + 90 * 8;
3224 if (CompareStackPointer)
3225 ScratchReg =
IsLP64 ? X86::RSP : X86::ESP;
3248 TlsOffset = 0x48 + 90 * 4;
3261 if (CompareStackPointer)
3262 ScratchReg = X86::ESP;
3264 BuildMI(checkMBB,
DL,
TII.get(X86::LEA32r), ScratchReg)
3283 unsigned ScratchReg2;
3285 if (CompareStackPointer) {
3288 SaveScratch2 =
false;
3300 "Scratch register is live-in and not saved");
3306 BuildMI(checkMBB,
DL,
TII.get(X86::MOV32ri), ScratchReg2)
3317 BuildMI(checkMBB,
DL,
TII.get(X86::POP32r), ScratchReg2);
3333 const unsigned RegAX =
IsLP64 ? X86::RAX : X86::EAX;
3334 const unsigned Reg10 =
IsLP64 ? X86::R10 : X86::R10D;
3335 const unsigned Reg11 =
IsLP64 ? X86::R11 : X86::R11D;
3336 const unsigned MOVrr =
IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3372 "code model and thunks not yet implemented.");
3389 BuildMI(allocMBB,
DL,
TII.get(X86::MORESTACK_RET_RESTORE_R10));
3398#ifdef EXPENSIVE_CHECKS
3409 for (
int i = 0, e = HiPELiteralsMD->
getNumOperands(); i != e; ++i) {
3411 if (
Node->getNumOperands() != 2)
3413 MDString *NodeName = dyn_cast<MDString>(
Node->getOperand(0));
3415 if (!NodeName || !NodeVal)
3418 if (ValConst && NodeName->
getString() == LiteralName) {
3424 " required but not provided");
3435 return MI.isMetaInstruction();
3461 assert(&(*MF.
begin()) == &PrologueMBB &&
"Shrink-wrapping not supported yet");
3466 if (!HiPELiteralsMD)
3468 "Can't generate HiPE prologue without runtime parameters");
3470 HiPELiteralsMD,
Is64Bit ?
"AMD64_LEAF_WORDS" :
"X86_LEAF_WORDS");
3471 const unsigned CCRegisteredArgs =
Is64Bit ? 6 : 5;
3472 const unsigned Guaranteed = HipeLeafWords *
SlotSize;
3479 "HiPE prologue is only supported on Linux operating systems.");
3489 unsigned MoreStackForCalls = 0;
3491 for (
auto &
MBB : MF) {
3492 for (
auto &
MI :
MBB) {
3512 if (
F->getName().contains(
"erlang.") ||
F->getName().contains(
"bif_") ||
3516 unsigned CalleeStkArity =
F->arg_size() > CCRegisteredArgs
3517 ?
F->arg_size() - CCRegisteredArgs
3519 if (HipeLeafWords - 1 > CalleeStkArity)
3521 std::max(MoreStackForCalls,
3522 (HipeLeafWords - 1 - CalleeStkArity) *
SlotSize);
3525 MaxStack += MoreStackForCalls;
3530 if (MaxStack > Guaranteed) {
3534 for (
const auto &LI : PrologueMBB.
liveins()) {
3542 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3543 unsigned LEAop, CMPop, CALLop;
3548 LEAop = X86::LEA64r;
3549 CMPop = X86::CMP64rm;
3550 CALLop = X86::CALL64pcrel32;
3554 LEAop = X86::LEA32r;
3555 CMPop = X86::CMP32rm;
3556 CALLop = X86::CALLpcrel32;
3561 "HiPE prologue scratch register is live-in");
3568 PReg,
false, SPLimitOffset);
3578 PReg,
false, SPLimitOffset);
3588#ifdef EXPENSIVE_CHECKS
3605 if (NumPops != 1 && NumPops != 2)
3613 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask())
3617 unsigned FoundRegs = 0;
3623 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
3625 for (
auto Candidate : RegClass) {
3629 if (!
RegMask.clobbersPhysReg(Candidate))
3633 if (
MRI.isReserved(Candidate))
3638 if (MO.isReg() && MO.isDef() &&
3639 TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) {
3648 Regs[FoundRegs++] = Candidate;
3649 if (FoundRegs == (
unsigned)NumPops)
3657 while (FoundRegs < (
unsigned)NumPops)
3658 Regs[FoundRegs++] = Regs[0];
3660 for (
int i = 0; i < NumPops; ++i)
3671 unsigned Opcode =
I->getOpcode();
3672 bool isDestroy =
Opcode ==
TII.getCallFrameDestroyOpcode();
3685 if (!reserveCallFrame) {
3706 bool HasDwarfEHHandlers = !WindowsCFI && !MF.
getLandingPads().empty();
3708 if (HasDwarfEHHandlers && !isDestroy &&
3718 Amount -= InternalAmt;
3728 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
3730 if (StackAdjustment) {
3737 if (StackAdjustment) {
3738 if (!(
F.hasMinSize() &&
3739 adjustStackWithPops(
MBB, InsertPos,
DL, StackAdjustment)))
3740 BuildStackAdjustment(
MBB, InsertPos,
DL, StackAdjustment,
3753 int64_t CfaAdjustment = -StackAdjustment;
3756 if (CfaAdjustment) {
3769 while (CI !=
B && !std::prev(CI)->isCall())
3771 BuildStackAdjustment(
MBB, CI,
DL, -InternalAmt,
false);
3787 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
3823 bool CompactUnwind =
3843 "restoring EBP/ESI on non-32-bit target");
3855 int EHRegSize = MFI.getObjectSize(FI);
3860 X86::EBP,
true, -EHRegSize)
3866 int EndOffset = -EHRegOffset - EHRegSize;
3879 "end of registration object above normal EBP position!");
3880 }
else if (UsedReg == BasePtr) {
3890 assert(UsedReg == BasePtr);
3916 FrameBase.
Kind = DwarfFrameBase::CFA;
3922 return DwarfFrameBase{DwarfFrameBase::Register, {FrameRegister}};
3927struct X86FrameSortingObject {
3928 bool IsValid =
false;
3929 unsigned ObjectIndex = 0;
3930 unsigned ObjectSize = 0;
3932 unsigned ObjectNumUses = 0;
3948struct X86FrameSortingComparator {
3949 inline bool operator()(
const X86FrameSortingObject &
A,
3950 const X86FrameSortingObject &
B)
const {
3951 uint64_t DensityAScaled, DensityBScaled;
3971 DensityAScaled =
static_cast<uint64_t>(
A.ObjectNumUses) *
3973 DensityBScaled =
static_cast<uint64_t>(
B.ObjectNumUses) *
3984 if (DensityAScaled == DensityBScaled)
3985 return A.ObjectAlignment <
B.ObjectAlignment;
3987 return DensityAScaled < DensityBScaled;
4001 if (ObjectsToAllocate.
empty())
4013 for (
auto &Obj : ObjectsToAllocate) {
4014 SortingObjects[Obj].IsValid =
true;
4015 SortingObjects[Obj].ObjectIndex = Obj;
4019 if (ObjectSize == 0)
4021 SortingObjects[Obj].ObjectSize = 4;
4023 SortingObjects[Obj].ObjectSize = ObjectSize;
4027 for (
auto &
MBB : MF) {
4028 for (
auto &
MI :
MBB) {
4029 if (
MI.isDebugInstr())
4035 int Index = MO.getIndex();
4039 SortingObjects[
Index].IsValid)
4040 SortingObjects[
Index].ObjectNumUses++;
4055 for (
auto &Obj : SortingObjects) {
4059 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4063 if (!
TRI->hasStackRealignment(MF) &&
hasFP(MF))
4064 std::reverse(ObjectsToAllocate.
begin(), ObjectsToAllocate.
end());
4076 Offset += getWinEHFuncletFrameSize(MF);
4096 adjustFrameForMsvcCxxEh(MF);
4100void X86FrameLowering::adjustFrameForMsvcCxxEh(
MachineFunction &MF)
const {
4108 int64_t MinFixedObjOffset = -
SlotSize;
4114 int FrameIndex =
H.CatchObj.FrameIndex;
4115 if (FrameIndex != INT_MAX) {
4118 MinFixedObjOffset -= std::abs(MinFixedObjOffset) %
Align;
4126 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8;
4127 int64_t UnwindHelpOffset = MinFixedObjOffset -
SlotSize;
4154 MI->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
static bool isFuncletReturnInstr(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const uint64_t kSplitStackAvailable
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Given that RA is a live value
const HexagonInstrInfo * TII
static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static bool isTailCallOpcode(unsigned Opc)
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static bool is64Bit(const char *name)
static unsigned calculateSetFPREG(uint64_t SPAdjust)
static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary)
GetScratchRegister - Get a temp register for performing work in the segmented stack and the Erlang/Hi...
static unsigned getADDriOpcode(bool IsLP64)
static unsigned getPUSH2Opcode(const X86Subtarget &ST)
static unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm)
static unsigned getLEArOpcode(bool IsLP64)
static unsigned getSUBriOpcode(bool IsLP64)
static bool flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB)
Check if the flags need to be preserved before the terminators.
static unsigned getANDriOpcode(bool IsLP64, int64_t Imm)
static bool isEAXLiveIn(MachineBasicBlock &MBB)
static unsigned getADDrrOpcode(bool IsLP64)
static bool HasNestArgument(const MachineFunction *MF)
static unsigned getPOPOpcode(const X86Subtarget &ST)
static unsigned getPOP2Opcode(const X86Subtarget &ST)
static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD, const StringRef LiteralName)
Lookup an ERTS parameter in the !hipe.literals named metadata node.
static bool blockEndIsUnreachable(const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI)
static unsigned getSUBrrOpcode(bool IsLP64)
static unsigned getPUSHOpcode(const X86Subtarget &ST)
static const unsigned FramePtr
static constexpr uint32_t RegMask
static constexpr uint32_t Opcode
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
reverse_iterator rend() const
bool empty() const
empty - Check if the array is empty.
reverse_iterator rbegin() const
LLVM Basic Block Representation.
iterator_range< const_set_bits_iterator > set_bits() const