40#define DEBUG_TYPE "x86-fl"
42STATISTIC(NumFrameLoopProbe,
"Number of loop stack probes used in prologue");
44 "Number of extra stack probes generated in prologue");
52 STI(STI),
TII(*STI.getInstrInfo()),
TRI(STI.getRegisterInfo()) {
76 (
hasFP(MF) && !
TRI->hasStackRealignment(MF)) ||
109 return IsLP64 ? X86::SUB64ri32 : X86::SUB32ri;
113 return IsLP64 ? X86::ADD64ri32 : X86::ADD32ri;
117 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
121 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
125 return IsLP64 ? X86::AND64ri32 : X86::AND32ri;
129 return IsLP64 ? X86::LEA64r : X86::LEA32r;
135 return X86::MOV32ri64;
137 return X86::MOV64ri32;
145 unsigned Reg = RegMask.PhysReg;
147 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
148 Reg == X86::AH || Reg == X86::AL)
162 bool BreakNext =
false;
167 if (Reg != X86::EFLAGS)
188 if (Succ->isLiveIn(X86::EFLAGS))
199 int64_t NumBytes,
bool InEpilogue)
const {
200 bool isSub = NumBytes < 0;
214 if (EmitInlineStackProbe && !InEpilogue) {
220 }
else if (
Offset > Chunk) {
231 unsigned AddSubRROpc =
240 MI->getOperand(3).setIsDead();
242 }
else if (
Offset > 8 * Chunk) {
266 MI->getOperand(3).setIsDead();
288 ? (
Is64Bit ? X86::PUSH64r : X86::PUSH32r)
289 : (
Is64Bit ? X86::POP64r : X86::POP32r);
298 BuildStackAdjustment(
MBB,
MBBI,
DL, isSub ? -ThisVal : ThisVal, InEpilogue)
308 assert(
Offset != 0 &&
"zero offset stack adjustment requested");
326 if (UseLEA && !
STI.useLeaForSP())
331 "We shouldn't have allowed this insertion point");
348 MI->getOperand(3).setIsDead();
355 bool doMergeWithPrevious)
const {
374 if (doMergeWithPrevious && PI !=
MBB.
begin() && PI->isCFIInstruction())
377 unsigned Opc = PI->getOpcode();
380 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD32ri) &&
381 PI->getOperand(0).getReg() ==
StackPtr) {
383 Offset = PI->getOperand(2).getImm();
384 }
else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
385 PI->getOperand(0).getReg() ==
StackPtr &&
386 PI->getOperand(1).getReg() ==
StackPtr &&
387 PI->getOperand(2).getImm() == 1 &&
388 PI->getOperand(3).getReg() == X86::NoRegister &&
389 PI->getOperand(5).getReg() == X86::NoRegister) {
391 Offset = PI->getOperand(4).getImm();
392 }
else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB32ri) &&
393 PI->getOperand(0).getReg() ==
StackPtr) {
395 Offset = -PI->getOperand(2).getImm();
400 if (PI !=
MBB.
end() && PI->isCFIInstruction()) {
407 if (!doMergeWithPrevious)
444 unsigned DwarfReg =
MRI->getDwarfRegNum(MachineFramePtr,
true);
468 unsigned DwarfReg =
MRI->getDwarfRegNum(Reg,
true);
479 CfaExpr.
push_back(dwarf::DW_CFA_expression);
488 unsigned DwarfFramePtr =
MRI->getDwarfRegNum(MachineFramePtr,
true);
489 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
504 int FI =
MI->getOperand(1).getIndex();
512 unsigned DwarfFramePtr =
MRI->getDwarfRegNum(MachineFramePtr,
true);
513 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
519 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
529void X86FrameLowering::emitZeroCallUsedRegs(
BitVector RegsToZero,
545 if (!X86::RFP80RegClass.
contains(Reg))
548 unsigned NumFPRegs = ST.is64Bit() ? 8 : 7;
549 for (
unsigned i = 0; i != NumFPRegs; ++i)
552 for (
unsigned i = 0; i != NumFPRegs; ++i)
560 if (
TRI->isGeneralPurposeRegister(MF, Reg)) {
562 RegsToZero.
reset(Reg);
572 if (
ST.hasMMX() && X86::VR64RegClass.contains(Reg))
577 if (X86::VR128RegClass.
contains(Reg)) {
582 }
else if (X86::VR256RegClass.
contains(Reg)) {
586 XorOp = X86::VPXORrr;
587 }
else if (X86::VR512RegClass.
contains(Reg)) {
591 XorOp = X86::VPXORYrr;
592 }
else if (X86::VK1RegClass.
contains(Reg) ||
599 XorOp =
ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr;
613 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum)
const {
620 emitStackProbeInline(MF,
MBB,
MBBI,
DL,
false);
623 emitStackProbeCall(MF,
MBB,
MBBI,
DL, InProlog, InstrNum);
634 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
636 if (Where != PrologMBB.
end()) {
638 emitStackProbeInline(MF, PrologMBB, Where,
DL,
true);
639 Where->eraseFromParent();
647 bool InProlog)
const {
650 emitStackProbeInlineWindowsCoreCLR64(MF,
MBB,
MBBI,
DL, InProlog);
652 emitStackProbeInlineGeneric(MF,
MBB,
MBBI,
DL, InProlog);
655void X86FrameLowering::emitStackProbeInlineGeneric(
664 "different expansion expected for CoreCLR 64 bit");
666 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
667 uint64_t ProbeChunk = StackProbeSize * 8;
670 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
675 if (
Offset > ProbeChunk) {
677 MaxAlign % StackProbeSize);
680 MaxAlign % StackProbeSize);
684void X86FrameLowering::emitStackProbeInlineGenericBlock(
689 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
693 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
694 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
698 assert(AlignOffset < StackProbeSize);
701 if (StackProbeSize <
Offset + AlignOffset) {
704 BuildStackAdjustment(
MBB,
MBBI,
DL, -StackAdjustment,
false)
706 if (!HasFP && NeedsDwarfCFI) {
717 NumFrameExtraProbe++;
718 CurrentOffset = StackProbeSize - AlignOffset;
724 while (CurrentOffset + StackProbeSize <
Offset) {
725 BuildStackAdjustment(
MBB,
MBBI,
DL, -StackProbeSize,
false)
728 if (!HasFP && NeedsDwarfCFI) {
738 NumFrameExtraProbe++;
739 CurrentOffset += StackProbeSize;
748 unsigned Opc =
Is64Bit ? X86::PUSH64r : X86::PUSH32r;
753 BuildStackAdjustment(
MBB,
MBBI,
DL, -ChunkSize,
false)
760void X86FrameLowering::emitStackProbeInlineGenericLoop(
768 "Inline stack probe loop will clobber live EFLAGS.");
770 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
774 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
775 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
778 if (AlignOffset < StackProbeSize) {
780 BuildStackAdjustment(
MBB,
MBBI,
DL, -AlignOffset,
false)
788 NumFrameExtraProbe++;
801 MF.
insert(MBBIter, testMBB);
802 MF.
insert(MBBIter, tailMBB);
823 if (!HasFP && NeedsDwarfCFI) {
826 const Register DwarfFinalStackProbed =
833 nullptr,
TRI->getDwarfRegNum(DwarfFinalStackProbed,
true)));
840 BuildStackAdjustment(*testMBB, testMBB->
end(),
DL, -StackProbeSize,
874 BuildStackAdjustment(*tailMBB, TailMBBIter,
DL, -TailOffset,
880 if (!HasFP && NeedsDwarfCFI) {
890 nullptr,
TRI->getDwarfRegNum(DwarfStackPtr,
true)));
898void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
902 assert(
STI.is64Bit() &&
"different expansion needed for 32 bit");
909 "Inline stack probe loop will clobber live EFLAGS.");
944 MF.
insert(MBBIter, RoundMBB);
945 MF.
insert(MBBIter, LoopMBB);
946 MF.
insert(MBBIter, ContinueMBB);
954 const int64_t ThreadEnvironmentStackLimit = 0x10;
956 const int64_t PageMask = ~(
PageSize - 1);
962 const Register SizeReg = InProlog ? X86::RAX
963 :
MRI.createVirtualRegister(RegClass),
964 ZeroReg = InProlog ? X86::RCX
965 :
MRI.createVirtualRegister(RegClass),
966 CopyReg = InProlog ? X86::RDX
967 :
MRI.createVirtualRegister(RegClass),
968 TestReg = InProlog ? X86::RDX
969 :
MRI.createVirtualRegister(RegClass),
970 FinalReg = InProlog ? X86::RDX
971 :
MRI.createVirtualRegister(RegClass),
972 RoundedReg = InProlog ? X86::RDX
973 :
MRI.createVirtualRegister(RegClass),
974 LimitReg = InProlog ? X86::RCX
975 :
MRI.createVirtualRegister(RegClass),
976 JoinReg = InProlog ? X86::RCX
977 :
MRI.createVirtualRegister(RegClass),
978 ProbeReg = InProlog ? X86::RCX
979 :
MRI.createVirtualRegister(RegClass);
982 int64_t RCXShadowSlot = 0;
983 int64_t RDXShadowSlot = 0;
999 int64_t InitSlot = 8 + CalleeSaveSize + (
HasFP ? 8 : 0);
1003 RCXShadowSlot = InitSlot;
1005 RDXShadowSlot = InitSlot;
1006 if (IsRDXLiveIn && IsRCXLiveIn)
1047 .
addImm(ThreadEnvironmentStackLimit)
1055 BuildMI(RoundMBB,
DL,
TII.get(X86::AND64ri32), RoundedReg)
1096 TII.get(X86::MOV64rm), X86::RCX),
1097 X86::RSP,
false, RCXShadowSlot);
1100 TII.get(X86::MOV64rm), X86::RDX),
1101 X86::RSP,
false, RDXShadowSlot);
1107 BuildMI(*ContinueMBB, ContinueMBBI,
DL,
TII.get(X86::SUB64rr), X86::RSP)
1120 for (++BeforeMBBI; BeforeMBBI !=
MBB.
end(); ++BeforeMBBI) {
1136void X86FrameLowering::emitStackProbeCall(
1139 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum)
const {
1145 "code model and indirect thunks not yet implemented.");
1149 "Stack probe calls will clobber live EFLAGS.");
1153 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1155 CallOp = X86::CALLpcrel32;
1216 for (++ExpansionMBBI; ExpansionMBBI !=
MBBI; ++ExpansionMBBI)
1224 const uint64_t Win64MaxSEHOffset = 128;
1225 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
1227 return SEHFrameOffset & -16;
1241 MaxAlign = (
StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1248 MaxAlign = (MaxAlign > 16) ? MaxAlign :
Align(16);
1250 MaxAlign =
Align(16);
1252 return MaxAlign.
value();
1265 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1266 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1271 if (Reg ==
StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1273 NumFrameLoopProbe++;
1284 MF.
insert(MBBIter, entryMBB);
1285 MF.
insert(MBBIter, headMBB);
1286 MF.
insert(MBBIter, bodyMBB);
1287 MF.
insert(MBBIter, footMBB);
1288 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1297 BuildMI(entryMBB,
DL,
TII.get(TargetOpcode::COPY), FinalStackProbed)
1301 BuildMI(entryMBB,
DL,
TII.get(AndOp), FinalStackProbed)
1302 .
addReg(FinalStackProbed)
1307 MI->getOperand(3).setIsDead();
1311 .
addReg(FinalStackProbed)
1325 const unsigned SUBOpc =
1335 .
addReg(FinalStackProbed)
1356 const unsigned SUBOpc =
1366 .
addReg(FinalStackProbed)
1382 .
addReg(FinalStackProbed)
1404 MI->getOperand(3).setIsDead();
1412 "MF used frame lowering for wrong subtarget");
1421bool X86FrameLowering::isWin64Prologue(
const MachineFunction &MF)
const {
1425bool X86FrameLowering::needsDwarfCFI(
const MachineFunction &MF)
const {
1517 "MF used frame lowering for wrong subtarget");
1523 uint64_t MaxAlign = calculateMaxStackAlign(MF);
1529 bool FnHasClrFunclet =
1531 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1532 bool HasFP =
hasFP(MF);
1533 bool IsWin64Prologue = isWin64Prologue(MF);
1538 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1539 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1545 bool HasWinCFI =
false;
1556 ArgBaseReg =
MI->getOperand(0).getReg();
1569 if (NeedsDwarfCFI) {
1571 unsigned DwarfStackPtr =
TRI->getDwarfRegNum(ArgBaseReg,
true);
1590 if (TailCallArgReserveSize && IsWin64Prologue)
1593 const bool EmitStackProbeCall =
1610 .
addUse(X86::NoRegister);
1657 !EmitStackProbeCall &&
1664 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
1671 if (TailCallArgReserveSize != 0) {
1672 BuildStackAdjustment(
MBB,
MBBI,
DL, -(
int)TailCallArgReserveSize,
1695 Register Establisher = X86::NoRegister;
1701 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1717 NumBytes = FrameSize -
1721 if (
TRI->hasStackRealignment(MF) && !IsWin64Prologue)
1722 NumBytes =
alignTo(NumBytes, MaxAlign);
1729 if (NeedsDwarfCFI && !ArgBaseReg.
isValid()) {
1735 nullptr, -2 * stackGrowth + (
int)TailCallArgReserveSize),
1739 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1743 (
int)TailCallArgReserveSize),
1761 if (Attrs.hasAttrSomewhere(Attribute::SwiftAsync)) {
1796 if (!IsWin64Prologue && !IsFunclet) {
1805 if (NeedsDwarfCFI) {
1808 CfaExpr.
push_back(dwarf::DW_CFA_expression);
1810 unsigned DwarfReg =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1813 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
1822 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1841 assert(!IsFunclet &&
"funclets without FPs not yet implemented");
1842 NumBytes = StackSize -
1849 if (HasFP &&
TRI->hasStackRealignment(MF))
1857 unsigned ParentFrameNumBytes = NumBytes;
1859 NumBytes = getWinEHFuncletFrameSize(MF);
1862 bool PushedRegs =
false;
1867 (
MBBI->getOpcode() == X86::PUSH32r ||
1868 MBBI->getOpcode() == X86::PUSH64r)) {
1873 if (!HasFP && NeedsDwarfCFI) {
1894 if (!IsWin64Prologue && !IsFunclet &&
TRI->hasStackRealignment(MF) &&
1896 assert(HasFP &&
"There should be a frame pointer if stack is realigned.");
1922 uint64_t AlignedNumBytes = NumBytes;
1923 if (IsWin64Prologue && !IsFunclet &&
TRI->hasStackRealignment(MF))
1924 AlignedNumBytes =
alignTo(AlignedNumBytes, MaxAlign);
1925 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
1927 "The Red Zone is not accounted for in stack probes");
1949 int64_t
Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
1957 .
addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
1976 }
else if (NumBytes) {
1980 if (NeedsWinCFI && NumBytes) {
1987 int SEHFrameOffset = 0;
1988 unsigned SPOrEstablisher;
1995 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
1999 Establisher,
false, PSPSlotOffset)
2006 false, PSPSlotOffset)
2013 SPOrEstablisher = Establisher;
2018 if (IsWin64Prologue && HasFP) {
2025 SPOrEstablisher,
false, SEHFrameOffset);
2028 .
addReg(SPOrEstablisher);
2031 if (NeedsWinCFI && !IsFunclet) {
2032 assert(!NeedsWinFPO &&
"this setframe incompatible with FPO data");
2041 }
else if (IsFunclet &&
STI.is32Bit()) {
2065 if (X86::FR64RegClass.
contains(Reg)) {
2068 if (IsWin64Prologue && IsFunclet)
2076 assert(!NeedsWinFPO &&
"SEH_SaveXMM incompatible with FPO data");
2086 if (NeedsWinCFI && HasWinCFI)
2090 if (FnHasClrFunclet && !IsFunclet) {
2094 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2108 if (IsWin64Prologue &&
TRI->hasStackRealignment(MF)) {
2109 assert(HasFP &&
"There should be a frame pointer if stack is realigned.");
2110 BuildStackAlignAND(
MBB,
MBBI,
DL, SPOrEstablisher, MaxAlign);
2114 if (IsFunclet &&
STI.is32Bit())
2147 assert(UsedReg == BasePtr);
2156 int FI =
MI->getOperand(1).getIndex();
2157 unsigned MOVmr =
Is64Bit ? X86::MOV64mr : X86::MOV32mr;
2164 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2166 if (!HasFP && NumBytes) {
2207 switch (
MI.getOpcode()) {
2209 case X86::CLEANUPRET:
2231X86FrameLowering::getPSPSlotOffsetFromSP(
const MachineFunction &MF)
const {
2238 return static_cast<unsigned>(
Offset);
2242X86FrameLowering::getWinEHFuncletFrameSize(
const MachineFunction &MF)
const {
2248 unsigned XMMSize = WinEHXMMSlotInfo.
size() *
2249 TRI->getSpillSize(X86::VR128RegClass);
2258 UsedSize = getPSPSlotOffsetFromSP(MF) +
SlotSize;
2269 return FrameSizeMinusRBP + XMMSize - CSSize;
2273 return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi ||
2274 Opc == X86::TCRETURNmi ||
2275 Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNdi64 ||
2276 Opc == X86::TCRETURNmi64;
2287 DL =
MBBI->getDebugLoc();
2295 bool NeedsWin64CFI =
2301 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2304 bool HasFP =
hasFP(MF);
2313 unsigned Opc = X86::LEA32r;
2315 ArgBaseReg =
MI->getOperand(0).getReg();
2316 if (
STI.is64Bit()) {
2318 StackReg = X86::RSP;
2329 if (NeedsDwarfCFI) {
2330 unsigned DwarfStackPtr =
TRI->getDwarfRegNum(StackReg,
true);
2340 assert(HasFP &&
"EH funclets without FP not yet implemented");
2341 NumBytes = getWinEHFuncletFrameSize(MF);
2345 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2349 if (
TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2350 NumBytes =
alignTo(FrameSize, MaxAlign);
2352 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2354 uint64_t SEHStackAllocAmt = NumBytes;
2379 if (NeedsDwarfCFI) {
2381 unsigned DwarfStackPtr =
2382 TRI->getDwarfRegNum(
Is64Bit ? X86::RSP : X86::ESP,
true);
2388 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
2403 unsigned Opc = PI->getOpcode();
2405 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2419 int FI =
MI->getOperand(1).getIndex();
2420 unsigned MOVrm =
Is64Bit ? X86::MOV64rm : X86::MOV32rm;
2427 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2428 emitCatchRetReturnValue(
MBB, FirstCSPop, &*Terminator);
2431 DL =
MBBI->getDebugLoc();
2443 if (
TRI->hasStackRealignment(MF))
2447 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2459 if (LEAAmount != 0) {
2470 }
else if (NumBytes) {
2473 if (!HasFP && NeedsDwarfCFI) {
2477 nullptr, CSSize + TailCallArgReserveSize +
SlotSize),
2492 if (!HasFP && NeedsDwarfCFI) {
2499 unsigned Opc = PI->getOpcode();
2501 if (Opc == X86::POP32r || Opc == X86::POP64r) {
2519 assert(
Offset >= 0 &&
"TCDelta should never be positive");
2543 else if (
TRI->hasStackRealignment(MF))
2557 int64_t FPDelta = 0;
2568 if (IsWin64Prologue) {
2576 uint64_t NumBytes = FrameSize - CSSize;
2586 FPDelta = FrameSize - SEHFrameOffset;
2588 "FPDelta isn't aligned per the Win64 ABI!");
2600 if (TailCallReturnAddrDelta < 0)
2601 Offset -= TailCallReturnAddrDelta;
2619 const auto it = WinEHXMMSlotInfo.find(FI);
2621 if (it == WinEHXMMSlotInfo.end())
2632 int Adjustment)
const {
2642 bool IgnoreSPUpdates)
const {
2692 "we don't handle this case!");
2724 std::vector<CalleeSavedInfo> &CSI)
const {
2728 unsigned CalleeSavedFrameSize = 0;
2729 unsigned XMMCalleeSavedFrameSize = 0;
2735 if (TailCallReturnAddrDelta < 0) {
2746 TailCallReturnAddrDelta -
SlotSize,
true);
2750 if (this->TRI->hasBasePointer(MF)) {
2776 for (
unsigned i = 0; i < CSI.size(); ++i) {
2777 if (
TRI->regsOverlap(CSI[i].getReg(),FPReg)) {
2778 CSI.erase(CSI.begin() + i);
2818 MVT VT = MVT::Other;
2819 if (X86::VK16RegClass.
contains(Reg))
2820 VT =
STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2823 unsigned Size =
TRI->getSpillSize(*RC);
2824 Align Alignment =
TRI->getSpillAlign(*RC);
2826 assert(SpillSlotOffset < 0 &&
"SpillSlotOffset should always < 0 on X86");
2827 SpillSlotOffset = -
alignTo(-SpillSlotOffset, Alignment);
2830 SpillSlotOffset -=
Size;
2836 if (X86::VR128RegClass.
contains(Reg)) {
2837 WinEHXMMSlotInfo[
SlotIndex] = XMMCalleeSavedFrameSize;
2838 XMMCalleeSavedFrameSize +=
Size;
2857 unsigned Opc =
STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2865 bool isLiveIn =
MRI.isLiveIn(Reg);
2870 bool CanKill = !isLiveIn;
2874 if (
MRI.isLiveIn(*AReg)) {
2892 unsigned Opc =
STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2893 Register BaseReg = this->TRI->getBaseRegister();
2907 MVT VT = MVT::Other;
2908 if (X86::VK16RegClass.
contains(Reg))
2909 VT =
STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2931 "SEH should not use CATCHRET");
2936 if (
STI.is64Bit()) {
2968 if (
MI->getOpcode() == X86::CATCHRET) {
2982 if (X86::GR64RegClass.
contains(Reg) ||
2987 MVT VT = MVT::Other;
2988 if (X86::VK16RegClass.
contains(Reg))
2989 VT =
STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
3000 unsigned Opc =
STI.is64Bit() ? X86::POP64r : X86::POP32r;
3001 Register BaseReg = this->TRI->getBaseRegister();
3007 unsigned Opc =
STI.is64Bit() ? X86::POP64r : X86::POP32r;
3010 if (!X86::GR64RegClass.
contains(Reg) &&
3030 SavedRegs.
set(BasePtr);
3039 if (
I->hasNestAttr() && !
I->use_empty())
3056 return Primary ? X86::R14 : X86::R13;
3058 return Primary ? X86::EBX : X86::EDI;
3063 return Primary ? X86::R11 : X86::R12;
3065 return Primary ? X86::R11D : X86::R12D;
3075 "nested function.");
3076 return Primary ? X86::EAX : X86::ECX;
3079 return Primary ? X86::EDX : X86::EAX;
3080 return Primary ? X86::ECX : X86::EAX;
3091 unsigned TlsReg, TlsOffset;
3096 assert(&(*MF.
begin()) == &PrologueMBB &&
"Shrink-wrapping not supported yet");
3100 "Scratch register is live-in");
3120 bool IsNested =
false;
3129 for (
const auto &LI : PrologueMBB.
liveins()) {
3148 TlsOffset =
IsLP64 ? 0x70 : 0x40;
3151 TlsOffset = 0x60 + 90*8;
3165 if (CompareStackPointer)
3166 ScratchReg =
IsLP64 ? X86::RSP : X86::ESP;
3179 TlsOffset = 0x48 + 90*4;
3192 if (CompareStackPointer)
3193 ScratchReg = X86::ESP;
3205 unsigned ScratchReg2;
3207 if (CompareStackPointer) {
3210 SaveScratch2 =
false;
3222 "Scratch register is live-in and not saved");
3228 BuildMI(checkMBB,
DL,
TII.get(X86::MOV32ri), ScratchReg2)
3237 BuildMI(checkMBB,
DL,
TII.get(X86::POP32r), ScratchReg2);
3251 const unsigned RegAX =
IsLP64 ? X86::RAX : X86::EAX;
3252 const unsigned Reg10 =
IsLP64 ? X86::R10 : X86::R10D;
3253 const unsigned Reg11 =
IsLP64 ? X86::R11 : X86::R11D;
3254 const unsigned MOVrr =
IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3291 "code model and thunks not yet implemented.");
3308 BuildMI(allocMBB,
DL,
TII.get(X86::MORESTACK_RET_RESTORE_R10));
3317#ifdef EXPENSIVE_CHECKS
3328 for (
int i = 0, e = HiPELiteralsMD->
getNumOperands(); i != e; ++i) {
3330 if (
Node->getNumOperands() != 2)
continue;
3331 MDString *NodeName = dyn_cast<MDString>(
Node->getOperand(0));
3333 if (!NodeName || !NodeVal)
continue;
3335 if (ValConst && NodeName->
getString() == LiteralName) {
3341 +
" required but not provided");
3352 return MI.isMetaInstruction();
3378 assert(&(*MF.
begin()) == &PrologueMBB &&
"Shrink-wrapping not supported yet");
3383 if (!HiPELiteralsMD)
3385 "Can't generate HiPE prologue without runtime parameters");
3386 const unsigned HipeLeafWords
3388 Is64Bit ?
"AMD64_LEAF_WORDS" :
"X86_LEAF_WORDS");
3389 const unsigned CCRegisteredArgs =
Is64Bit ? 6 : 5;
3390 const unsigned Guaranteed = HipeLeafWords *
SlotSize;
3396 "HiPE prologue is only supported on Linux operating systems.");
3406 unsigned MoreStackForCalls = 0;
3408 for (
auto &
MBB : MF) {
3409 for (
auto &
MI :
MBB) {
3429 if (
F->getName().contains(
"erlang.") ||
F->getName().contains(
"bif_") ||
3433 unsigned CalleeStkArity =
3434 F->arg_size() > CCRegisteredArgs ?
F->arg_size()-CCRegisteredArgs : 0;
3435 if (HipeLeafWords - 1 > CalleeStkArity)
3436 MoreStackForCalls = std::max(MoreStackForCalls,
3437 (HipeLeafWords - 1 - CalleeStkArity) *
SlotSize);
3440 MaxStack += MoreStackForCalls;
3445 if (MaxStack > Guaranteed) {
3449 for (
const auto &LI : PrologueMBB.
liveins()) {
3457 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3458 unsigned LEAop, CMPop, CALLop;
3463 LEAop = X86::LEA64r;
3464 CMPop = X86::CMP64rm;
3465 CALLop = X86::CALL64pcrel32;
3469 LEAop = X86::LEA32r;
3470 CMPop = X86::CMP32rm;
3471 CALLop = X86::CALLpcrel32;
3476 "HiPE prologue scratch register is live-in");
3480 SPReg,
false, -MaxStack);
3483 .
addReg(ScratchReg), PReg,
false, SPLimitOffset);
3488 addExternalSymbol(
"inc_stack_0");
3490 SPReg,
false, -MaxStack);
3492 .
addReg(ScratchReg), PReg,
false, SPLimitOffset);
3500#ifdef EXPENSIVE_CHECKS
3517 if (NumPops != 1 && NumPops != 2)
3525 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask())
3529 unsigned FoundRegs = 0;
3535 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
3537 for (
auto Candidate : RegClass) {
3545 if (
MRI.isReserved(Candidate))
3550 if (MO.isReg() && MO.isDef() &&
3551 TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) {
3560 Regs[FoundRegs++] = Candidate;
3561 if (FoundRegs == (
unsigned)NumPops)
3569 while (FoundRegs < (
unsigned)NumPops)
3570 Regs[FoundRegs++] = Regs[0];
3572 for (
int i = 0; i < NumPops; ++i)
3574 TII.get(
STI.is64Bit() ? X86::POP64r : X86::POP32r), Regs[i]);
3583 unsigned Opcode =
I->getOpcode();
3584 bool isDestroy = Opcode ==
TII.getCallFrameDestroyOpcode();
3597 if (!reserveCallFrame) {
3618 bool HasDwarfEHHandlers = !WindowsCFI && !MF.
getLandingPads().empty();
3620 if (HasDwarfEHHandlers && !isDestroy &&
3630 Amount -= InternalAmt;
3640 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
3642 if (StackAdjustment) {
3649 if (StackAdjustment) {
3650 if (!(
F.hasMinSize() &&
3651 adjustStackWithPops(
MBB, InsertPos,
DL, StackAdjustment)))
3652 BuildStackAdjustment(
MBB, InsertPos,
DL, StackAdjustment,
3665 int64_t CfaAdjustment = -StackAdjustment;
3668 if (CfaAdjustment) {
3681 while (CI !=
B && !std::prev(CI)->isCall())
3683 BuildStackAdjustment(
MBB, CI,
DL, -InternalAmt,
false);
3699 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
3735 bool CompactUnwind =
3755 "restoring EBP/ESI on non-32-bit target");
3766 int FI =
FuncInfo.EHRegNodeFrameIndex;
3772 X86::EBP,
true, -EHRegSize)
3778 int EndOffset = -EHRegOffset - EHRegSize;
3779 FuncInfo.EHRegNodeEndOffset = EndOffset;
3791 "end of registration object above normal EBP position!");
3792 }
else if (UsedReg == BasePtr) {
3802 assert(UsedReg == BasePtr);
3828 FrameBase.
Kind = DwarfFrameBase::CFA;
3834 return DwarfFrameBase{DwarfFrameBase::Register, {FrameRegister}};
3839struct X86FrameSortingObject {
3840 bool IsValid =
false;
3841 unsigned ObjectIndex = 0;
3842 unsigned ObjectSize = 0;
3844 unsigned ObjectNumUses = 0;
3860struct X86FrameSortingComparator {
3861 inline bool operator()(
const X86FrameSortingObject &
A,
3862 const X86FrameSortingObject &
B)
const {
3863 uint64_t DensityAScaled, DensityBScaled;
3883 DensityAScaled =
static_cast<uint64_t>(
A.ObjectNumUses) *
3885 DensityBScaled =
static_cast<uint64_t>(
B.ObjectNumUses) *
3896 if (DensityAScaled == DensityBScaled)
3897 return A.ObjectAlignment <
B.ObjectAlignment;
3899 return DensityAScaled < DensityBScaled;
3913 if (ObjectsToAllocate.
empty())
3925 for (
auto &Obj : ObjectsToAllocate) {
3926 SortingObjects[Obj].IsValid =
true;
3927 SortingObjects[Obj].ObjectIndex = Obj;
3931 if (ObjectSize == 0)
3933 SortingObjects[Obj].ObjectSize = 4;
3935 SortingObjects[Obj].ObjectSize = ObjectSize;
3939 for (
auto &
MBB : MF) {
3940 for (
auto &
MI :
MBB) {
3941 if (
MI.isDebugInstr())
3947 int Index = MO.getIndex();
3951 SortingObjects[
Index].IsValid)
3952 SortingObjects[
Index].ObjectNumUses++;
3967 for (
auto &Obj : SortingObjects) {
3971 ObjectsToAllocate[i++] = Obj.ObjectIndex;
3975 if (!
TRI->hasStackRealignment(MF) &&
hasFP(MF))
3976 std::reverse(ObjectsToAllocate.
begin(), ObjectsToAllocate.
end());
3988 Offset += getWinEHFuncletFrameSize(MF);
4008 adjustFrameForMsvcCxxEh(MF);
4012void X86FrameLowering::adjustFrameForMsvcCxxEh(
MachineFunction &MF)
const {
4020 int64_t MinFixedObjOffset = -
SlotSize;
4026 int FrameIndex =
H.CatchObj.FrameIndex;
4027 if (FrameIndex != INT_MAX) {
4030 MinFixedObjOffset -= std::abs(MinFixedObjOffset) %
Align;
4038 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8;
4039 int64_t UnwindHelpOffset = MinFixedObjOffset -
SlotSize;
4066 MI->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
static bool isFuncletReturnInstr(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const uint64_t kSplitStackAvailable
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Given that RA is a live value
const HexagonInstrInfo * TII
static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static bool isTailCallOpcode(unsigned Opc)
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static bool is64Bit(const char *name)
static unsigned calculateSetFPREG(uint64_t SPAdjust)
static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary)
GetScratchRegister - Get a temp register for performing work in the segmented stack and the Erlang/Hi...
static unsigned getADDriOpcode(bool IsLP64)
static unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm)
static unsigned getLEArOpcode(bool IsLP64)
static unsigned getSUBriOpcode(bool IsLP64)
static bool flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB)
Check if the flags need to be preserved before the terminators.
static unsigned getANDriOpcode(bool IsLP64, int64_t Imm)
static bool isEAXLiveIn(MachineBasicBlock &MBB)
static unsigned getADDrrOpcode(bool IsLP64)
static bool HasNestArgument(const MachineFunction *MF)
static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD, const StringRef LiteralName)
Lookup an ERTS parameter in the !hipe.literals named metadata node.
static bool blockEndIsUnreachable(const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI)
static unsigned getSUBrrOpcode(bool IsLP64)
static const unsigned FramePtr
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
iterator_range< const_set_bits_iterator > set_bits() const
static BranchProbability getOne()
static BranchProbability getZero()
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
bool hasPersonalityFn() const
Check whether this function has a personality function.
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset)
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createRestore(MCSymbol *L, unsigned Register)
.cfi_restore says that the rule for Register is now the same as it was at the beginning of the functi...
static MCCFIInstruction createGnuArgsSize(MCSymbol *L, int Size)
A special wrapper for .cfi_escape that indicates GNU_ARGS_SIZE.
static MCCFIInstruction createDefCfaRegister(MCSymbol *L, unsigned Register)
.cfi_def_cfa_register modifies a rule for computing CFA.
OpType getOperation() const
static MCCFIInstruction createAdjustCfaOffset(MCSymbol *L, int Adjustment)
.cfi_adjust_cfa_offset Same as .cfi_def_cfa_offset, but Offset is a relative value that is added/subt...
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset)
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset)
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
const MCObjectFileInfo * getObjectFileInfo() const
const MCRegisterInfo * getRegisterInfo() const
MCSection * getCompactUnwindSection() const
MCRegAliasIterator enumerates all registers aliasing Reg.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
StringRef getString() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator_range< livein_iterator > liveins() const
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI, MCRegister Reg, const_iterator Before, unsigned Neighborhood=10) const
Return whether (physical) register Reg has been defined and not killed as of just before Before.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
iterator getFirstNonPHI()
Returns a pointer to the