266#define DEBUG_TYPE "frame-info"
273 bool IsTailCallReturn = (
MBB.end() !=
MBBI)
277 int64_t ArgumentPopSize = 0;
278 if (IsTailCallReturn) {
284 ArgumentPopSize = StackAdjust.
getImm();
293 return ArgumentPopSize;
304 if (LastI !=
MBB.end() && (LastI->getOpcode() == AArch64::CATCHRET ||
305 LastI->getOpcode() == AArch64::CLEANUPRET))
351 if (AFI->hasCalculatedStackSizeSVE())
380bool AArch64FrameLowering::homogeneousPrologEpilog(
384 const AArch64Options &CLOpts =
386 if (!CLOpts.homogeneous_prolog_epilog)
408 if (AFI->hasSwiftAsyncContext() || AFI->hasStreamingModeChanges())
415 unsigned NumGPRs = 0;
416 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
418 if (Reg == AArch64::LR) {
419 assert(CSRegs[
I + 1] == AArch64::FP);
420 if (NumGPRs % 2 != 0)
432bool AArch64FrameLowering::producePairRegisters(
MachineFunction &MF)
const {
451 if (
MI.isDebugInstr() ||
MI.isPseudo() ||
452 MI.getOpcode() == AArch64::ADDXri ||
453 MI.getOpcode() == AArch64::ADDSXri)
478 bool IsWin64,
bool IsFunclet)
const {
480 "Tail call reserved stack must be aligned to 16 bytes");
483 }
else if (IsFunclet) {
497 SmallSetVector<int, 8> CatchObjFrameIndices;
498 for (
const WinEHTryBlockMapEntry &TBME : EHInfo.
TryBlockMap) {
501 if ((FrameIndex != INT_MAX) &&
502 CatchObjFrameIndices.
insert(FrameIndex)) {
503 FixedObjectSize =
alignTo(FixedObjectSize,
510 FixedObjectSize += 8;
512 return alignTo(FixedObjectSize, 16);
523 const unsigned RedZoneSize =
536 bool LowerQRegCopyThroughMem =
Subtarget.hasFPARMv8() &&
540 return !(MFI.
hasCalls() ||
hasFP(MF) || NumBytes > RedZoneSize ||
568 if (
Subtarget.getTargetLowering()->useStackGuardMixFP())
577 RegInfo->hasStackRealignment(MF))
623 if (TT.isOSDarwin() || TT.isOSWindowsOrUEFI())
661 unsigned Opc =
I->getOpcode();
662 bool IsDestroy =
Opc ==
TII->getCallFrameDestroyOpcode();
663 uint64_t CalleePopAmount = IsDestroy ?
I->getOperand(1).getImm() : 0;
666 int64_t Amount =
I->getOperand(0).getImm();
674 if (CalleePopAmount == 0) {
685 assert(Amount > -0xffffff && Amount < 0xffffff &&
"call frame too large");
687 if (
TLI->hasInlineStackProbe(MF) &&
696 "non-reserved call frame without var sized objects?");
705 }
else if (CalleePopAmount != 0) {
708 assert(CalleePopAmount < 0xffffff &&
"call frame too large");
726 CFIBuilder.buildDefCFA(AArch64::SP, 0);
729 if (MFI.shouldSignReturnAddress(MF)) {
730 if (MFI.branchProtectionPAuthLR()) {
731 CFIBuilder.buildNegateRAStateWithPC();
733 CFIBuilder.buildNegateRAState();
738 if (MFI.needsShadowCallStackPrologueEpilogue(MF))
739 CFIBuilder.buildSameValue(AArch64::X18);
742 const std::vector<CalleeSavedInfo> &CSI =
744 for (
const auto &Info : CSI) {
746 if (!
TRI.regNeedsCFI(Reg, Reg))
748 CFIBuilder.buildSameValue(Reg);
761 case AArch64::W##n: \
762 case AArch64::X##n: \
787 case AArch64::B##n: \
788 case AArch64::H##n: \
789 case AArch64::S##n: \
790 case AArch64::D##n: \
791 case AArch64::Q##n: \
792 return HasSVE ? AArch64::Z##n : AArch64::Q##n
829void AArch64FrameLowering::emitZeroCallUsedRegs(
BitVector RegsToZero,
841 const AArch64Subtarget &STI = MF.
getSubtarget<AArch64Subtarget>();
844 BitVector GPRsToZero(
TRI.getNumRegs());
845 BitVector FPRsToZero(
TRI.getNumRegs());
849 bool HasFPR = STI.hasFPARMv8();
851 if (
TRI.isGeneralPurposeRegister(MF,
Reg)) {
854 GPRsToZero.set(XReg);
858 FPRsToZero.set(XReg);
865 for (MCRegister
Reg : GPRsToZero.set_bits())
869 for (MCRegister
Reg : FPRsToZero.set_bits())
873 for (MCRegister PReg :
874 {AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
875 AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
876 AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
878 if (RegsToZero[PReg])
884bool AArch64FrameLowering::windowsRequiresStackProbe(
887 const AArch64FunctionInfo &MFI = *MF.
getInfo<AArch64FunctionInfo>();
891 StackSizeInBytes >=
uint64_t(MFI.getStackProbeSize());
900 for (
unsigned i = 0; CSRegs[i]; ++i)
906 bool HasCall)
const {
918 LivePhysRegs LiveRegs(
TRI);
921 LiveRegs.addReg(AArch64::X16);
922 LiveRegs.addReg(AArch64::X17);
923 LiveRegs.addReg(AArch64::X18);
928 if (LiveRegs.available(
MRI, AArch64::X9))
931 for (
unsigned Reg : AArch64::GPR64RegClass) {
932 if (LiveRegs.available(
MRI,
Reg))
965 MBB.isLiveIn(AArch64::NZCV))
968 if (RegInfo->hasStackRealignment(*MF) ||
TLI->hasInlineStackProbe(*MF))
969 if (!findScratchNonCalleeSaveRegister(TmpMBB).
isValid())
975 windowsRequiresStackProbe(*MF, std::numeric_limits<uint64_t>::max()))
976 if (!findScratchNonCalleeSaveRegister(TmpMBB,
true).
isValid())
993 if (!
F.needsUnwindTableEntry())
997 if (
F.hasPersonalityFn())
1005 if (
F.hasFnAttribute(Attribute::Naked))
1017bool AArch64FrameLowering::shouldSignReturnAddressEverywhere(
1033 unsigned Opc =
MBBI->getOpcode();
1037 unsigned ImmIdx =
MBBI->getNumOperands() - 1;
1038 int Imm =
MBBI->getOperand(ImmIdx).getImm();
1046 case AArch64::STR_ZXI:
1047 case AArch64::LDR_ZXI: {
1048 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1055 case AArch64::STR_PXI:
1056 case AArch64::LDR_PXI: {
1057 unsigned Reg0 = RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1064 case AArch64::LDPDpost:
1067 case AArch64::STPDpre: {
1068 unsigned Reg0 = RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1069 unsigned Reg1 = RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1070 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFRegP_X))
1077 case AArch64::LDPXpost:
1080 case AArch64::STPXpre: {
1083 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1084 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFPLR_X))
1088 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveRegP_X))
1089 .
addImm(RegInfo->getSEHRegNum(Reg0))
1090 .
addImm(RegInfo->getSEHRegNum(Reg1))
1095 case AArch64::LDRDpost:
1098 case AArch64::STRDpre: {
1099 unsigned Reg = RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1100 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFReg_X))
1106 case AArch64::LDRXpost:
1109 case AArch64::STRXpre: {
1110 unsigned Reg = RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1117 case AArch64::STPDi:
1118 case AArch64::LDPDi: {
1119 unsigned Reg0 = RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1120 unsigned Reg1 = RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1128 case AArch64::STPXi:
1129 case AArch64::LDPXi: {
1133 int SEHReg0 = RegInfo->getSEHRegNum(Reg0);
1134 int SEHReg1 = RegInfo->getSEHRegNum(Reg1);
1136 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1140 else if (SEHReg0 >= 19 && SEHReg1 >= 19)
1147 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegIP))
1154 case AArch64::STRXui:
1155 case AArch64::LDRXui: {
1156 int Reg = RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1163 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegI))
1169 case AArch64::STRDui:
1170 case AArch64::LDRDui: {
1171 unsigned Reg = RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1178 case AArch64::STRQui:
1179 case AArch64::LDRQui: {
1180 unsigned Reg = RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1181 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQ))
1187 case AArch64::STPQi:
1188 case AArch64::LDPQi: {
1189 unsigned Reg0 = RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1190 unsigned Reg1 = RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1191 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQP))
1198 case AArch64::LDRQpost:
1201 case AArch64::STRQpre: {
1202 unsigned Reg = RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1203 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQX))
1209 case AArch64::LDPQpost:
1212 case AArch64::STPQpre: {
1213 unsigned Reg0 = RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1214 unsigned Reg1 = RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1215 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQPX))
1234 if (ST.isTargetDarwin())
1256 DL =
MBBI->getDebugLoc();
1258 TII->createPauthEpilogueInstr(
MBB,
DL);
1262 EmitSignRA(MF.
front());
1264 if (
MBB.isEHFuncletEntry())
1266 if (
MBB.isReturnBlock())
1322 StackOffset SVEStackSize = ZPRStackSize + PPRStackSize;
1327 if (MFI.isVariableSizedObjectIndex(FI)) {
1337 if (MFI.hasScalableStackID(FI)) {
1338 if (FPAfterSVECalleeSaves &&
1341 "split-sve-objects not supported with FPAfterSVECalleeSaves");
1349 AccessOffset = -PPRStackSize;
1350 return AccessOffset +
1355 bool IsFixed = MFI.isFixedObjectIndex(FI);
1360 if (!IsFixed && !IsCSR) {
1361 ScalableOffset = -SVEStackSize;
1362 }
else if (FPAfterSVECalleeSaves && IsCSR) {
1377 int64_t ObjectOffset)
const {
1381 bool IsWin64 =
Subtarget.isCallingConvWin64(
F.getCallingConv(),
F.isVarArg());
1382 unsigned FixedObject =
1383 getFixedObjectSize(MF, AFI, IsWin64,
false);
1391 int64_t ObjectOffset)
const {
1402 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
1403 ? getFPOffset(MF, ObjectOffset).getFixed()
1404 : getStackOffset(MF, ObjectOffset).getFixed();
1409 bool ForSimm)
const {
1411 int64_t ObjectOffset = MFI.getObjectOffset(FI);
1412 bool isFixed = MFI.isFixedObjectIndex(FI);
1415 FrameReg, PreferFP, ForSimm);
1421 bool ForSimm)
const {
1427 int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
1428 int64_t
Offset = getStackOffset(MF, ObjectOffset).getFixed();
1436 const int64_t FixedObjectSize =
1437 getFixedObjectSize(MF, AFI, IsWin64,
false);
1441 bool isSVE = MFI.isScalableStackID(StackID);
1445 StackOffset SVEStackSize = ZPRStackSize + PPRStackSize;
1456 PreferFP &= !SVEStackSize;
1464 }
else if (isCSR && RegInfo->hasStackRealignment(MF)) {
1468 assert(
hasFP(MF) &&
"Re-aligned stack must have frame pointer");
1470 }
else if (
hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
1475 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
1476 PreferFP |=
Offset > -FPOffset && !SVEStackSize;
1478 if (FPOffset >= 0) {
1482 }
else if (MFI.hasVarSizedObjects()) {
1486 bool CanUseBP = RegInfo->hasBasePointer(MF);
1487 if (FPOffsetFits && CanUseBP)
1494 }
else if (MF.
hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1501 "Funclets should only be present on Win64");
1505 if (FPOffsetFits && PreferFP)
1512 ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
1513 "In the presence of dynamic stack pointer realignment, "
1514 "non-argument/CSR objects cannot be accessed through the frame pointer");
1531 FPOffset -= PPRStackSize;
1533 SPOffset -= PPRStackSize;
1538 if (FPAfterSVECalleeSaves) {
1549 RegInfo->hasStackRealignment(MF))) {
1550 FrameReg = RegInfo->getFrameRegister(MF);
1553 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
1560 if (FPAfterSVECalleeSaves) {
1567 SVEAreaOffset = SVECalleeSavedStack;
1569 SVEAreaOffset = SVECalleeSavedStack - SVEStackSize;
1572 SVEAreaOffset = SVEStackSize;
1574 SVEAreaOffset = SVEStackSize - SVECalleeSavedStack;
1577 if (UseFP && !(isFixed || isCSR))
1578 SVEAreaOffset = -SVEStackSize;
1579 if (!UseFP && (isFixed || isCSR))
1580 SVEAreaOffset = SVEStackSize;
1584 FrameReg = RegInfo->getFrameRegister(MF);
1589 if (RegInfo->hasBasePointer(MF))
1590 FrameReg = RegInfo->getBaseRegister();
1592 assert(!MFI.hasVarSizedObjects() &&
1593 "Can't use SP when we have var sized objects.");
1594 FrameReg = AArch64::SP;
1621 !(
Subtarget.getTargetLowering()->supportSwiftError() &&
1622 Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
1628 int PairOffset,
unsigned Reg1,
1629 unsigned Reg2,
bool NeedsWinCFI,
1638 if (Reg2 == AArch64::FP)
1648 if (
TRI->getEncodingValue(Reg2) ==
TRI->getEncodingValue(Reg1) + 1)
1649 return SpillExtendedVolatile
1650 ? !((Reg1 == AArch64::FP && Reg2 == AArch64::LR) ||
1651 (PairOffset % 16) == 0)
1656 if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
1657 (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR)
1667 int PairOffset,
unsigned Reg1,
1668 unsigned Reg2,
bool UsesWinAAPCS,
1669 bool NeedsWinCFI,
bool NeedsFrameRecord,
1673 Reg1, Reg2, NeedsWinCFI,
TRI);
1677 if (NeedsFrameRecord)
1678 return Reg2 == AArch64::LR;
1687 int64_t MinOff, MaxOff;
1688 TypeSize ScaleValue(0U,
false), Width(0U,
false);
1689 if (!
TII->getMemOpInfo(Opcode, ScaleValue, Width, MinOff, MaxOff))
1692 if (
Offset % ScaleValue.getKnownMinValue() != 0)
1695 Offset /= ScaleValue.getKnownMinValue();
1701struct RegGroupInfo {
1706 enum RegType { GPR, FPR64, FPR128, PPR, ZPR, VG }
Type;
1709 RegGroupInfo() =
default;
1711 bool isPaired()
const {
return getNumRegs() == 2; }
1713 bool isQuad()
const {
return getNumRegs() == 4; }
1715 bool isGrouped()
const {
return isPaired() || isQuad(); }
1717 unsigned getNumRegs()
const {
1723 unsigned NumRegs = (Reg1 - Reg2) + 1;
1724 assert(NumRegs == 2 || NumRegs == 4);
1728 bool isScalable()
const {
return Type == PPR ||
Type == ZPR; }
1732 return {Reg1, Reg2};
1734 assert(Reg1 - 3 == Reg2 &&
"Quad not contiguous");
1735 return {Reg1, Reg1 - 1, Reg1 - 2, Reg1 - 3};
1740 auto getFrameIndexes()
const {
1741 return seq(FrameIdx, FrameIdx +
int(getNumRegs()));
1744 Register getAsMultiVectorRegister()
const {
1745 assert(
Type == ZPR && isGrouped() &&
"Expected ZPR register group");
1746 return (isQuad() ? AArch64::Z0_Z1_Z2_Z3 : AArch64::Z0_Z1) +
1747 (Reg2 - AArch64::Z0);
1750 void dump(
const TargetRegisterInfo *
TRI)
const {
1754 dbgs() <<
") -> fi#(";
1756 [&](
int FrameIdx) {
dbgs() << FrameIdx; });
1764 for (
unsigned PReg = AArch64::P8; PReg <= AArch64::P15; ++PReg) {
1765 if (SavedRegs.
test(PReg)) {
1766 unsigned PNReg = PReg - AArch64::P0 + AArch64::PN0;
1776 if (
Subtarget.getCLOpts().disable_multivector_spill_fill)
1780 bool IsLocallyStreaming =
1788 (!IsLocallyStreaming &&
Subtarget.isStreaming()));
1796 bool NeedsFrameRecord) {
1816 (
Count & 1) == 0) &&
1817 "Odd number of callee-saved regs to spill!");
1819 int StackFillDir = -1;
1821 unsigned FirstReg = 0;
1829 FirstReg =
Count - 1;
1841 bool SpillExtendedVolatile =
1843 const auto &
Reg = CSI.getReg();
1844 return Reg >= AArch64::X0 &&
Reg <= AArch64::X18;
1847 int ZPRByteOffset = 0;
1848 int PPRByteOffset = 0;
1853 }
else if (!FPAfterSVECalleeSaves) {
1864 auto AlignOffset = [StackFillDir](
int Offset,
int Align) {
1865 if (StackFillDir < 0)
1871 for (
unsigned i = FirstReg; i <
Count; i += RegInc) {
1873 RGI.Reg1 = CSI[i].getReg();
1875 if (AArch64::GPR64RegClass.
contains(RGI.Reg1)) {
1876 RGI.Type = RegGroupInfo::GPR;
1877 RGI.RC = &AArch64::GPR64RegClass;
1878 }
else if (AArch64::FPR64RegClass.
contains(RGI.Reg1)) {
1879 RGI.Type = RegGroupInfo::FPR64;
1880 RGI.RC = &AArch64::FPR64RegClass;
1881 }
else if (AArch64::FPR128RegClass.
contains(RGI.Reg1)) {
1882 RGI.Type = RegGroupInfo::FPR128;
1883 RGI.RC = &AArch64::FPR128RegClass;
1884 }
else if (AArch64::ZPRRegClass.
contains(RGI.Reg1)) {
1885 RGI.Type = RegGroupInfo::ZPR;
1886 RGI.RC = &AArch64::ZPRRegClass;
1887 }
else if (AArch64::PPRRegClass.
contains(RGI.Reg1)) {
1888 RGI.Type = RegGroupInfo::PPR;
1889 RGI.RC = &AArch64::PPRRegClass;
1890 }
else if (RGI.Reg1 == AArch64::VG) {
1891 RGI.Type = RegGroupInfo::VG;
1892 RGI.RC = &AArch64::FIXED_REGSRegClass;
1897 int &ScalableByteOffset = RGI.Type == RegGroupInfo::PPR && SplitPPRs
1902 if (HasCSHazardPadding &&
1905 ByteOffset += StackFillDir * StackHazardSize;
1908 int Scale =
TRI->getSpillSize(*RGI.RC);
1910 if (
unsigned(i + RegInc) <
Count && !HasCSHazardPadding) {
1911 MCRegister NextReg = CSI[i + RegInc].getReg();
1912 int Aligned = AlignOffset(ByteOffset, Scale);
1913 int PairOffset = IsWindows ?
Aligned :
Aligned + StackFillDir * 2 * Scale;
1914 bool PairFitsImmRange =
1915 PairOffset / Scale >= -64 && PairOffset / Scale <= 63;
1917 case RegGroupInfo::GPR:
1918 case RegGroupInfo::FPR64:
1919 case RegGroupInfo::FPR128: {
1921 RGI.Type == RegGroupInfo::GPR ? AArch64::GPR64RegClass
1922 : RGI.Type == RegGroupInfo::FPR64 ? AArch64::FPR64RegClass
1923 : AArch64::FPR128RegClass;
1925 if (SpillRC.
contains(NextReg) && PairFitsImmRange &&
1927 RGI.Reg1, NextReg, IsWindows,
1928 NeedsWinCFI, NeedsFrameRecord,
TRI))
1932 case RegGroupInfo::PPR:
1934 case RegGroupInfo::ZPR: {
1946 auto GetZPRTupleLowestReg =
1947 [&](
unsigned NumRegs) -> std::optional<Register> {
1949 if (RGI.Reg1 % NumRegs != NumRegs - 1)
1950 return std::nullopt;
1953 if (i + RegInc * NumRegs >
Count)
1954 return std::nullopt;
1956 for (
unsigned Idx = 1; Idx < NumRegs; ++Idx) {
1958 Register ExpectedReg = RGI.Reg1 - Idx;
1959 if (
Reg != ExpectedReg)
1960 return std::nullopt;
1963 return RGI.Reg1 - NumRegs + 1;
1966 for (
unsigned TupleSize : {4, 2}) {
1967 if (
auto LowReg = GetZPRTupleLowestReg(TupleSize)) {
1970 TupleSize == 4 ? AArch64::LD1B_4Z_IMM : AArch64::LD1B_2Z_IMM;
1972 (ScalableByteOffset + StackFillDir * TupleSize * Scale);
1982 case RegGroupInfo::VG:
1993 [[maybe_unused]]
bool RegsOrdered =
1994 all_of(
seq(RGI.getNumRegs() - 1), [&](
int RegOffset) {
1995 return CSI[i + RegInc * RegOffset].getFrameIdx() + RegInc ==
1996 CSI[i + RegInc * RegOffset + RegInc].getFrameIdx();
1998 assert(RegsOrdered &&
"Out of order callee saved regs!");
2000 assert((!RGI.isQuad() || RGI.Type == RegGroupInfo::ZPR) &&
2001 "Currently only ZPR's support four-register spills");
2003 assert((!RGI.isPaired() || !NeedsFrameRecord || RGI.Reg2 != AArch64::FP ||
2004 RGI.Reg1 == AArch64::LR) &&
2005 "FrameRecord must be allocated together with LR");
2008 assert((!RGI.isPaired() || !NeedsFrameRecord || RGI.Reg1 != AArch64::FP ||
2009 RGI.Reg2 == AArch64::LR) &&
2010 "FrameRecord must be allocated together with LR");
2018 ((RGI.Reg1 == AArch64::LR && RGI.Reg2 == AArch64::FP) ||
2019 RGI.Reg1 + 1 == RGI.Reg2))) &&
2020 "Callee-save registers not saved as adjacent register pair!");
2022 RGI.FrameIdx = CSI[i].getFrameIdx();
2025 RGI.FrameIdx = CSI[i + RegInc * (RGI.getNumRegs() - 1)].getFrameIdx();
2029 if (RGI.isScalable() && ScalableByteOffset % Scale != 0)
2030 ScalableByteOffset = AlignOffset(ScalableByteOffset, Scale);
2034 if (!RGI.isScalable() && ByteOffset % Scale != 0)
2035 ByteOffset = AlignOffset(ByteOffset, Scale);
2037 int OffsetPre = RGI.isScalable() ? ScalableByteOffset : ByteOffset;
2038 assert(OffsetPre % Scale == 0);
2040 if (RGI.isScalable())
2041 ScalableByteOffset += StackFillDir * RGI.getNumRegs() * Scale;
2043 ByteOffset += StackFillDir * RGI.getNumRegs() * Scale;
2048 ((!IsWindows && RGI.Reg2 == AArch64::FP) ||
2049 (IsWindows && RGI.Reg2 == AArch64::LR)))
2050 ByteOffset += StackFillDir * 8;
2054 if (NeedGapToAlignStack && !IsWindows && !RGI.isScalable() &&
2055 RGI.Type != RegGroupInfo::FPR128 && !RGI.isPaired() &&
2056 ByteOffset % 16 != 0) {
2057 ByteOffset += 8 * StackFillDir;
2063 NeedGapToAlignStack =
false;
2066 int OffsetPost = RGI.isScalable() ? ScalableByteOffset : ByteOffset;
2067 assert(OffsetPost % Scale == 0);
2070 int Offset = IsWindows ? OffsetPre : OffsetPost;
2075 ((!IsWindows && RGI.Reg2 == AArch64::FP) ||
2076 (IsWindows && RGI.Reg2 == AArch64::LR)))
2080 (!RGI.isGrouped() || RGI.Type != RegGroupInfo::ZPR ||
2082 TII, (RGI.isQuad() ? AArch64::LD1B_4Z_IMM : AArch64::LD1B_2Z_IMM),
2084 "Offset invalid for LD1B/ST1B pair/quad variant immediate");
2086 RGI.Offset =
Offset / Scale;
2087 assert((!RGI.isGrouped() || RGI.Type == RegGroupInfo::ZPR ||
2088 (!RGI.isScalable() && RGI.Offset >= -64 && RGI.Offset <= 63) ||
2089 (RGI.isScalable() && RGI.Offset >= -256 && RGI.Offset <= 255)) &&
2090 "Offset out of bounds for LDP/STP immediate");
2092 auto isFrameRecord = [&] {
2094 return IsWindows ? RGI.Reg1 == AArch64::FP && RGI.Reg2 == AArch64::LR
2095 : RGI.Reg1 == AArch64::LR && RGI.Reg2 == AArch64::FP;
2103 return i > 0 && RGI.Reg1 == AArch64::FP &&
2104 CSI[i - 1].getReg() == AArch64::LR;
2109 if (NeedsFrameRecord && isFrameRecord())
2113 i += RegInc * (RGI.getNumRegs() - 1);
2125 std::reverse(RegGroups.
begin(), RegGroups.
end());
2145 MRI.freezeReservedRegs();
2147 if (homogeneousPrologEpilog(MF)) {
2151 for (
auto &RGI : RegGroups) {
2156 if (!
MRI.isReserved(RGI.Reg1))
2157 MBB.addLiveIn(RGI.Reg1);
2158 if (RGI.isPaired() && !
MRI.isReserved(RGI.Reg2))
2159 MBB.addLiveIn(RGI.Reg2);
2163 bool PTrueCreated =
false;
2179 unsigned Size =
TRI->getSpillSize(*RGI.RC);
2180 Align Alignment =
TRI->getSpillAlign(*RGI.RC);
2182 case RegGroupInfo::GPR:
2183 StrOpc = RGI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
2185 case RegGroupInfo::FPR64:
2186 StrOpc = RGI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
2188 case RegGroupInfo::FPR128:
2189 StrOpc = RGI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2191 case RegGroupInfo::ZPR:
2192 StrOpc = RGI.isQuad()
2193 ? AArch64::ST1B_4Z_IMM
2194 : (RGI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI);
2196 case RegGroupInfo::PPR:
2197 StrOpc = AArch64::STR_PXI;
2199 case RegGroupInfo::VG:
2200 StrOpc = AArch64::STRXui;
2212 if (Reg1 == AArch64::VG) {
2214 Reg1 = findScratchNonCalleeSaveRegister(&
MBB,
true);
2225 return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
2226 AArch64::X0, LiveIn.PhysReg);
2234 RTLIB::Libcall LC = RTLIB::SMEABI_GET_CURRENT_VG;
2236 TRI->getCallPreservedMask(MF,
TLI.getLibcallCallingConv(LC));
2250 !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2251 "Windows unwdinding requires a consecutive (FP,LR) pair");
2256 if (!PTrueCreated && RGI.isGrouped() && RGI.isScalable()) {
2260 "Expects SVE2.1 or SME2 target and a predicate register");
2261 PTrueCreated =
true;
2268 if (!
MRI.isReserved(Reg))
2272 if (RGI.isGrouped() && RGI.isScalable()) {
2274 "Scalable register groups are not supported by Windows CFI yet");
2275#ifdef EXPENSIVE_CHECKS
2276 auto IsPPR = [](
const RegGroupInfo &c) {
2277 return c.Type == RegGroupInfo::PPR;
2279 auto PPRBegin = std::find_if(RegGroups.
begin(), RegGroups.
end(), IsPPR);
2280 auto IsZPR = [](
const RegGroupInfo &c) {
2281 return c.Type == RegGroupInfo::ZPR;
2283 auto ZPRBegin = std::find_if(RegGroups.
begin(), RegGroups.
end(), IsZPR);
2284 assert(!(PPRBegin < ZPRBegin) &&
2285 "Expected callee save predicate to be handled first");
2288 assert(RGI.Reg2 < RGI.Reg1 &&
"Expected reversed ZPR group");
2289 MIB.
addReg(RGI.getAsMultiVectorRegister());
2303 if (RGI.isPaired()) {
2322 if (RGI.Type == RegGroupInfo::ZPR)
2324 else if (RGI.Type == RegGroupInfo::PPR)
2342 DL =
MBBI->getDebugLoc();
2345 if (homogeneousPrologEpilog(MF, &
MBB)) {
2348 for (
auto &RGI : RegGroups) {
2356 auto IsPPR = [](
const RegGroupInfo &c) {
2357 return c.Type == RegGroupInfo::PPR;
2360 auto PPREnd = std::find_if_not(PPRBegin, RegGroups.
end(), IsPPR);
2361 std::reverse(PPRBegin, PPREnd);
2362 auto IsZPR = [](
const RegGroupInfo &c) {
2363 return c.Type == RegGroupInfo::ZPR;
2366 auto ZPREnd = std::find_if_not(ZPRBegin, RegGroups.
end(), IsZPR);
2367 std::reverse(ZPRBegin, ZPREnd);
2374 bool PTrueCreated =
false;
2375 for (
const RegGroupInfo &RGI : RegGroups) {
2388 unsigned Size =
TRI->getSpillSize(*RGI.RC);
2389 Align Alignment =
TRI->getSpillAlign(*RGI.RC);
2391 case RegGroupInfo::GPR:
2392 LdrOpc = RGI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2394 case RegGroupInfo::FPR64:
2395 LdrOpc = RGI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2397 case RegGroupInfo::FPR128:
2398 LdrOpc = RGI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2400 case RegGroupInfo::ZPR:
2401 LdrOpc = RGI.isQuad()
2402 ? AArch64::LD1B_4Z_IMM
2403 : (RGI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI);
2405 case RegGroupInfo::PPR:
2406 LdrOpc = AArch64::LDR_PXI;
2408 case RegGroupInfo::VG:
2418 if (!PTrueCreated && RGI.isGrouped() && RGI.isScalable()) {
2422 "Expects SVE2.1 or SME2 target and a predicate register");
2423 PTrueCreated =
true;
2428 if (KeepLR && RGI.Type == RegGroupInfo::GPR &&
2429 (Reg1 == AArch64::LR || Reg2 == AArch64::LR)) {
2433 if (!RGI.isPaired())
2435 assert(Reg2 == AArch64::LR &&
"Expected LR to be the second of the pair");
2451 if (RGI.isGrouped() && RGI.isScalable()) {
2453 "Scalable register groups are not supported by Windows CFI yet");
2454#ifdef EXPENSIVE_CHECKS
2455 assert(!(PPRBegin < ZPRBegin) &&
2456 "Expected callee save predicate to be handled first");
2458 assert(RGI.Reg2 < RGI.Reg1 &&
"Expected reversed ZPR group");
2498 return std::optional<int>(PSV->getFrameIndex());
2509 return std::nullopt;
2515 if (!
MI.mayLoadOrStore() ||
MI.getNumMemOperands() < 1)
2516 return std::nullopt;
2523 return AArch64::PPRRegClass.contains(
MI.getOperand(0).getReg());
2529void AArch64FrameLowering::determineStackHazardSlot(
2532 auto *AFI = MF.
getInfo<AArch64FunctionInfo>();
2533 if (StackHazardSize == 0 || StackHazardSize % 16 != 0 ||
2539 const AArch64Options &CLOpts =
2541 if (!CLOpts.stack_hazard_in_non_streaming &&
2542 Attrs.hasNonStreamingInterfaceAndBody())
2550 return AArch64::FPR64RegClass.contains(Reg) ||
2551 AArch64::FPR128RegClass.contains(Reg) ||
2552 AArch64::ZPRRegClass.contains(Reg);
2555 return AArch64::PPRRegClass.contains(Reg);
2557 bool HasFPRStackObjects =
false;
2558 bool HasPPRStackObjects =
false;
2559 if (!HasFPRCSRs || CLOpts.split_sve_objects) {
2560 enum SlotType : uint8_t {
2571 for (
auto &
MBB : MF) {
2572 for (
auto &
MI :
MBB) {
2574 if (!FI || FI < 0 || FI >
int(SlotTypes.size()))
2581 ? SlotType::ZPRorFPR
2587 for (
int FI = 0; FI < int(SlotTypes.size()); ++FI) {
2588 HasFPRStackObjects |= SlotTypes[FI] == SlotType::ZPRorFPR;
2591 if (SlotTypes[FI] == SlotType::PPR) {
2593 HasPPRStackObjects =
true;
2598 if (HasFPRCSRs || HasFPRStackObjects) {
2601 << StackHazardSize <<
"\n");
2608 if (CLOpts.split_sve_objects) {
2612 LLVM_DEBUG(
dbgs() <<
"Using SplitSVEObjects for SVE CC function\n");
2618 LLVM_DEBUG(
dbgs() <<
"Determining if SplitSVEObjects should be used in "
2619 "non-SVE CC function...\n");
2626 <<
"Calling convention is not supported with SplitSVEObjects\n");
2630 if (!HasPPRCSRs && !HasPPRStackObjects) {
2632 dbgs() <<
"Not using SplitSVEObjects as no PPRs are on the stack\n");
2636 if (!HasFPRCSRs && !HasFPRStackObjects) {
2639 <<
"Not using SplitSVEObjects as no FPRs or ZPRs are on the stack\n");
2643 [[maybe_unused]]
const AArch64Subtarget &
Subtarget =
2644 MF.getSubtarget<AArch64Subtarget>();
2646 "Expected SVE to be available for PPRs");
2648 const TargetRegisterInfo *
TRI = MF.getSubtarget().getRegisterInfo();
2652 BitVector FPRZRegs(SavedRegs.
size());
2653 for (
size_t Reg = 0,
E = SavedRegs.
size(); HasFPRCSRs &&
Reg <
E; ++
Reg) {
2654 BitVector::reference RegBit = SavedRegs[
Reg];
2657 unsigned SubRegIdx = 0;
2659 SubRegIdx = AArch64::dsub;
2661 SubRegIdx = AArch64::zsub;
2668 TRI->getMatchingSuperReg(
Reg, SubRegIdx, &AArch64::ZPRRegClass);
2671 SavedRegs |= FPRZRegs;
2701 RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister() :
MCRegister();
2704 bool HasUnpairedGPR64 =
false;
2705 bool HasPairZReg =
false;
2706 BitVector UserReservedRegs = RegInfo->getUserReservedRegs(MF);
2707 BitVector ReservedRegs = RegInfo->getReservedRegs(MF);
2710 for (
unsigned i = 0; CSRegs[i]; ++i) {
2714 if (Reg == BasePointerReg)
2719 if (UserReservedRegs[Reg]) {
2720 SavedRegs.
reset(Reg);
2724 bool RegUsed = SavedRegs.
test(Reg);
2726 const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
2727 if (RegIsGPR64 || AArch64::FPR64RegClass.
contains(Reg) ||
2728 AArch64::FPR128RegClass.
contains(Reg)) {
2731 if (HasUnpairedGPR64)
2732 PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
2734 PairedReg = CSRegs[i ^ 1];
2741 if (RegIsGPR64 && !AArch64::GPR64RegClass.
contains(PairedReg)) {
2743 HasUnpairedGPR64 =
true;
2746 AArch64::GPR64RegClass.contains(Reg, PairedReg) ||
2747 AArch64::FPR64RegClass.contains(Reg, PairedReg) ||
2748 AArch64::FPR128RegClass.contains(Reg, PairedReg));
2751 if (AArch64::GPR64RegClass.
contains(Reg) && !ReservedRegs[Reg]) {
2752 UnspilledCSGPR = Reg;
2753 UnspilledCSGPRPaired = PairedReg;
2761 if (producePairRegisters(MF) && PairedReg.
isValid() &&
2762 !SavedRegs.
test(PairedReg)) {
2763 SavedRegs.
set(PairedReg);
2764 if (AArch64::GPR64RegClass.
contains(PairedReg) &&
2765 !ReservedRegs[PairedReg])
2766 ExtraCSSpill = PairedReg;
2769 HasPairZReg |= (AArch64::ZPRRegClass.contains(Reg, CSRegs[i ^ 1]) &&
2770 SavedRegs.
test(CSRegs[i ^ 1]));
2778 if (PnReg.isValid())
2784 SavedRegs.
set(AArch64::P8);
2789 "Predicate cannot be a reserved register");
2799 SavedRegs.
set(AArch64::X18);
2805 determineStackHazardSlot(MF, SavedRegs);
2808 unsigned CSStackSize = 0;
2809 unsigned ZPRCSStackSize = 0;
2810 unsigned PPRCSStackSize = 0;
2812 for (
unsigned Reg : SavedRegs.
set_bits()) {
2814 assert(RC &&
"expected register class!");
2815 auto SpillSize =
TRI->getSpillSize(*RC);
2816 bool IsZPR = AArch64::ZPRRegClass.contains(Reg);
2817 bool IsPPR = !IsZPR && AArch64::PPRRegClass.contains(Reg);
2819 ZPRCSStackSize += SpillSize;
2821 PPRCSStackSize += SpillSize;
2827 return SavedRegs.test(SuperReg);
2830 CSStackSize += SpillSize;
2837 unsigned NumSavedRegs = SavedRegs.
count();
2850 SavedRegs.
set(AArch64::LR);
2855 windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2856 SavedRegs.
set(AArch64::FP);
2857 SavedRegs.
set(AArch64::LR);
2861 dbgs() <<
"*** determineCalleeSaves\nSaved CSRs:";
2862 for (
unsigned Reg : SavedRegs.
set_bits())
2868 auto [ZPRLocalStackSize, PPRLocalStackSize] =
2870 uint64_t SVELocals = ZPRLocalStackSize + PPRLocalStackSize;
2871 uint64_t SVEStackSize =
2872 alignTo(ZPRCSStackSize + PPRCSStackSize + SVELocals, 16);
2873 bool CanEliminateFrame = (SavedRegs.
count() == 0) && !SVEStackSize;
2882 int64_t CalleeStackUsed = 0;
2885 if (FixedOff > CalleeStackUsed)
2886 CalleeStackUsed = FixedOff;
2890 bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
2891 CalleeStackUsed) > EstimatedStackSizeLimit;
2892 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2904 <<
" to get a scratch register.\n");
2905 SavedRegs.
set(UnspilledCSGPR);
2906 ExtraCSSpill = UnspilledCSGPR;
2911 if (producePairRegisters(MF)) {
2912 if (!UnspilledCSGPRPaired.
isValid()) {
2915 SavedRegs.
reset(UnspilledCSGPR);
2919 SavedRegs.
set(UnspilledCSGPRPaired);
2925 if (!ExtraCSSpill.
isValid() ||
2929 unsigned Size =
TRI->getSpillSize(RC);
2930 Align Alignment =
TRI->getSpillAlign(RC);
2932 RS->addScavengingFrameIndex(FI);
2933 LLVM_DEBUG(
dbgs() <<
"No available CS registers, allocated fi#" << FI
2934 <<
" as the emergency spill slot.\n");
2939 CSStackSize += 8 * (SavedRegs.
count() - NumSavedRegs);
2946 uint64_t AlignedCSStackSize =
alignTo(CSStackSize, 16);
2948 << EstimatedStackSize + AlignedCSStackSize <<
" bytes.\n");
2952 "Should not invalidate callee saved info");
2969 std::vector<CalleeSavedInfo> &CSI) {
2971 "ZPR callee-save reordering not supported on Windows");
2978 return AArch64::ZPRRegClass.contains(CS.getReg());
2982 ZPRSaves = ZPRSaves.
drop_until(IsZPR).take_while(IsZPR);
2984 if (ZPRSaves.
size() < 2)
2991 const int Scale =
RegInfo->getSpillSize(AArch64::ZPRRegClass);
2992 const int LowestOffset =
2993 (ZPRByteOffset / Scale) -
static_cast<int>(ZPRSaves.
size());
2995 int NumToAlign =
alignTo(LowestOffset, 4) - LowestOffset;
2996 if (NumToAlign >
int(ZPRSaves.
size()))
3000 [](
const auto &
A,
const auto &
B) {
return A.getReg() <
B.getReg(); });
3006 int ZPRsNeededForAlignment = NumToAlign;
3007 for (
size_t i = 0; i < ZPRSaves.
size();) {
3008 int ZPRsRemaining = int(ZPRSaves.
size()) - i;
3009 if (i + 3 < ZPRSaves.
size() &&
3010 (ZPRSaves[i].getReg() - AArch64::Z0) % 4 == 0 &&
3011 ZPRSaves[i].getReg() + 3 == ZPRSaves[i + 3].getReg() &&
3012 ZPRsRemaining - 4 >= ZPRsNeededForAlignment) {
3014 {ZPRSaves[i], ZPRSaves[i + 1], ZPRSaves[i + 2], ZPRSaves[i + 3]});
3021 bool MustSplitPairForSingle =
3022 ZPRsNeededForAlignment == 1 && Singles.empty() && ZPRsRemaining == 2;
3024 if (i + 1 < ZPRSaves.
size() &&
3025 (ZPRSaves[i].getReg() - AArch64::Z0) % 2 == 0 &&
3026 ZPRSaves[i].getReg() + 1 == ZPRSaves[i + 1].getReg() &&
3027 !MustSplitPairForSingle) {
3028 Pairs.
push_back({ZPRSaves[i], ZPRSaves[i + 1]});
3031 if (ZPRsNeededForAlignment != 1 || !Singles.empty())
3032 ZPRsNeededForAlignment -= 2;
3038 Singles.push_back(ZPRSaves[i++]);
3039 ZPRsNeededForAlignment--;
3045 int NumUnassigned = ZPRSaves.
size();
3051 ZPRSaves[--NumUnassigned] = CS;
3055 if (NumToAlign & 0b01)
3056 Emit(Singles.pop_back_val());
3057 if (NumToAlign & 0b10) {
3061 Emit({Singles.pop_back_val(), Singles.pop_back_val()});
3064 for (
const auto &Quad : Quads)
3067 for (
const auto &Pair : Pairs)
3070 for (
const auto &
Single : Singles)
3076 std::vector<CalleeSavedInfo> &CSI)
const {
3085 std::reverse(CSI.begin(), CSI.end());
3099 find_if(CSI, [](
auto &Info) {
return Info.getReg() == AArch64::LR; });
3100 if (It != CSI.end())
3101 CSI.insert(It, VGInfo);
3103 CSI.push_back(VGInfo);
3113 int HazardSlotIndex = std::numeric_limits<int>::max();
3114 for (
auto &CS : CSI) {
3122 assert(HazardSlotIndex == std::numeric_limits<int>::max() &&
3123 "Unexpected register order for hazard slot");
3125 LLVM_DEBUG(
dbgs() <<
"Created CSR Hazard at slot " << HazardSlotIndex
3131 unsigned Size = RegInfo->getSpillSize(*RC);
3132 Align Alignment(RegInfo->getSpillAlign(*RC));
3134 CS.setFrameIdx(FrameIdx);
3148 HazardSlotIndex == std::numeric_limits<int>::max()) {
3150 LLVM_DEBUG(
dbgs() <<
"Created CSR Hazard at slot " << HazardSlotIndex
3177 int &Min,
int &Max) {
3178 Min = std::numeric_limits<int>::max();
3179 Max = std::numeric_limits<int>::min();
3185 for (
auto &CS : CSI) {
3186 if (AArch64::ZPRRegClass.
contains(CS.getReg()) ||
3187 AArch64::PPRRegClass.contains(CS.getReg())) {
3188 assert((Max == std::numeric_limits<int>::min() ||
3189 Max + 1 == CS.getFrameIdx()) &&
3190 "SVE CalleeSaves are not consecutive");
3191 Min = std::min(Min, CS.getFrameIdx());
3192 Max = std::max(Max, CS.getFrameIdx());
3195 return Min != std::numeric_limits<int>::max();
3208 uint64_t &ZPRStackTop = SVEStack.ZPRStackSize;
3216 "SVE vectors should never be passed on the stack by value, only by "
3220 auto AllocateObject = [&](
int FI) {
3229 if (Alignment >
Align(16))
3231 "Alignment of scalable vectors > 16 bytes is not yet supported");
3234 StackTop =
alignTo(StackTop, Alignment);
3236 assert(StackTop < (
uint64_t)std::numeric_limits<int64_t>::max() &&
3237 "SVE StackTop far too large?!");
3239 int64_t
Offset = -int64_t(StackTop);
3247 int MinCSFrameIndex, MaxCSFrameIndex;
3249 for (
int FI = MinCSFrameIndex; FI <= MaxCSFrameIndex; ++FI)
3262 int StackProtectorFI = -1;
3266 ObjectsToAllocate.
push_back(StackProtectorFI);
3282 for (
unsigned FI : ObjectsToAllocate)
3297 "Upwards growing stack unsupported");
3312 int64_t CurrentOffset =
3316 int FrameIndex =
H.CatchObj.FrameIndex;
3317 if ((FrameIndex != INT_MAX) && MFI.
getObjectOffset(FrameIndex) == 0) {
3328 int64_t UnwindHelpOffset =
alignTo(CurrentOffset + 8,
Align(16));
3329 assert(UnwindHelpOffset == getFixedObjectSize(MF, AFI,
true,
3331 "UnwindHelpOffset must be at the start of the fixed object area");
3334 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
3344 RS->enterBasicBlockEnd(
MBB);
3346 Register DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
3347 assert(DstReg &&
"There must be a free register after frame setup");
3358struct TagStoreInstr {
3367 MachineBasicBlock *
MBB;
3368 MachineRegisterInfo *
MRI;
3377 StackOffset FrameRegOffset;
3381 std::optional<int64_t> FrameRegUpdate;
3383 unsigned FrameRegUpdateFlags;
3393 TagStoreEdit(MachineBasicBlock *
MBB,
bool ZeroData)
3394 :
MBB(
MBB), ZeroData(ZeroData) {
3400 void addInstruction(TagStoreInstr
I) {
3402 TagStores.
back().Offset + TagStores.
back().Size ==
I.Offset) &&
3403 "Non-adjacent tag store instructions.");
3406 void clear() { TagStores.
clear(); }
3411 const AArch64FrameLowering *TFI,
bool TryMergeSPUpdate);
3418 const int64_t kMinOffset = -256 * 16;
3419 const int64_t kMaxOffset = 255 * 16;
3422 int64_t BaseRegOffsetBytes = FrameRegOffset.
getFixed();
3423 if (BaseRegOffsetBytes < kMinOffset ||
3424 BaseRegOffsetBytes + (
Size -
Size % 32) > kMaxOffset ||
3428 BaseRegOffsetBytes % 16 != 0) {
3433 BaseRegOffsetBytes = 0;
3438 int64_t InstrSize = (
Size > 16) ? 32 : 16;
3441 ? (ZeroData ? AArch64::STZGi : AArch64::STGi)
3443 assert(BaseRegOffsetBytes % 16 == 0);
3447 .
addImm(BaseRegOffsetBytes / 16)
3451 if (BaseRegOffsetBytes == 0)
3453 BaseRegOffsetBytes += InstrSize;
3472 int64_t LoopSize =
Size;
3475 if (FrameRegUpdate && *FrameRegUpdate)
3476 LoopSize -= LoopSize % 32;
3478 TII->get(ZeroData ? AArch64::STZGloop_wback
3479 : AArch64::STGloop_wback))
3486 LoopI->
setFlags(FrameRegUpdateFlags);
3488 int64_t ExtraBaseRegUpdate =
3489 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.
getFixed() -
Size) : 0;
3490 LLVM_DEBUG(
dbgs() <<
"TagStoreEdit::emitLoop: LoopSize=" << LoopSize
3491 <<
", Size=" <<
Size
3492 <<
", ExtraBaseRegUpdate=" << ExtraBaseRegUpdate
3493 <<
", FrameRegUpdate=" << FrameRegUpdate
3494 <<
", FrameRegOffset.getFixed()="
3495 << FrameRegOffset.
getFixed() <<
"\n");
3496 if (LoopSize <
Size) {
3500 int64_t STGOffset = ExtraBaseRegUpdate + 16;
3501 assert(STGOffset % 16 == 0 && STGOffset >= -4096 && STGOffset <= 4080 &&
3502 "STG immediate out of range");
3504 TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
3511 }
else if (ExtraBaseRegUpdate) {
3513 int64_t AddSubOffset = std::abs(ExtraBaseRegUpdate);
3514 assert(AddSubOffset <= 4095 &&
"ADD/SUB immediate out of range");
3517 TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
3530 int64_t
Size, int64_t *TotalOffset) {
3532 if ((
MI.getOpcode() == AArch64::ADDXri ||
3533 MI.getOpcode() == AArch64::SUBXri) &&
3534 MI.getOperand(0).getReg() ==
Reg &&
MI.getOperand(1).getReg() ==
Reg) {
3536 int64_t
Offset =
MI.getOperand(2).getImm() << Shift;
3537 if (
MI.getOpcode() == AArch64::SUBXri)
3548 const int64_t kMaxOffset = 4080 - 16;
3550 const int64_t kMinOffset = -4095;
3551 if (PostOffset <= kMaxOffset && PostOffset >= kMinOffset &&
3552 PostOffset % 16 == 0) {
3563 for (
auto &TS : TSE) {
3567 if (
MI->memoperands_empty()) {
3571 MemRefs.
append(
MI->memoperands_begin(),
MI->memoperands_end());
3577 bool TryMergeSPUpdate) {
3578 if (TagStores.
empty())
3580 TagStoreInstr &FirstTagStore = TagStores[0];
3581 TagStoreInstr &LastTagStore = TagStores[TagStores.
size() - 1];
3582 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
3583 DL = TagStores[0].MI->getDebugLoc();
3587 *MF, FirstTagStore.Offset,
false ,
3591 FrameRegUpdate = std::nullopt;
3593 mergeMemRefs(TagStores, CombinedMemRefs);
3596 dbgs() <<
"Replacing adjacent STG instructions:\n";
3597 for (
const auto &Instr : TagStores) {
3606 if (TagStores.
size() < 2)
3608 emitUnrolled(InsertI);
3611 int64_t TotalOffset = 0;
3612 if (TryMergeSPUpdate) {
3620 if (UpdateI !=
MBB->
end() &&
3621 canMergeRegUpdate(UpdateI, FrameReg, FrameRegOffset.
getFixed() +
Size,
3625 UpdateInstr = &*UpdateI++;
3632 if (!UpdateInstr && TagStores.
size() < 2)
3636 FrameRegUpdate = TotalOffset;
3637 FrameRegUpdateFlags = UpdateInstr->
getFlags();
3644 for (
auto &TS : TagStores)
3645 TS.MI->eraseFromParent();
3649 int64_t &
Size,
bool &ZeroData) {
3653 unsigned Opcode =
MI.getOpcode();
3654 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
3655 Opcode == AArch64::STZ2Gi);
3657 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
3658 if (!
MI.getOperand(0).isDead() || !
MI.getOperand(1).isDead())
3660 if (!
MI.getOperand(2).isImm() || !
MI.getOperand(3).isFI())
3663 Size =
MI.getOperand(2).getImm();
3667 if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
3669 else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
3674 if (
MI.getOperand(0).getReg() != AArch64::SP || !
MI.getOperand(1).isFI())
3678 16 *
MI.getOperand(2).getImm();
3682static size_t countAvailableScavengerSlots(
LivePhysRegs &LiveRegs,
3687 return LiveRegs.available(
MRI,
Reg);
3690 size_t NumEmergencySlots = 0;
3692 NumEmergencySlots =
RS->getNumScavengingFrameIndices();
3694 return FreeGPRs + NumEmergencySlots;
3713 if (!isMergeableStackTaggingInstruction(
MI,
Offset,
Size, FirstZeroData))
3719 constexpr int kScanLimit = 10;
3722 NextI !=
E &&
Count < kScanLimit; ++NextI) {
3731 if (isMergeableStackTaggingInstruction(
MI,
Offset,
Size, ZeroData)) {
3732 if (ZeroData != FirstZeroData)
3740 if (!
MI.isTransient())
3749 if (
MI.mayLoadOrStore() ||
MI.hasUnmodeledSideEffects() ||
MI.isCall())
3765 LiveRegs.addLiveOuts(*
MBB);
3770 LiveRegs.stepBackward(*
I);
3773 if (LiveRegs.contains(AArch64::NZCV))
3784 dbgs() <<
"Failed to merge MTE stack tagging instructions into loop "
3785 <<
"due to high register pressure.\n");
3790 [](
const TagStoreInstr &
Left,
const TagStoreInstr &
Right) {
3795 int64_t CurOffset = Instrs[0].Offset;
3796 for (
auto &Instr : Instrs) {
3797 if (CurOffset >
Instr.Offset)
3804 TagStoreEdit TSE(
MBB, FirstZeroData);
3805 std::optional<int64_t> EndOffset;
3806 for (
auto &Instr : Instrs) {
3807 if (EndOffset && *EndOffset !=
Instr.Offset) {
3809 TSE.emitCode(InsertI, TFI,
false);
3813 TSE.addInstruction(Instr);
3831 .stack_tagging_merge_settag;
3835 II = tryMergeAdjacentSTG(
II,
this, RS);
3842 shouldSignReturnAddressEverywhere(MF))
3851 bool IgnoreSPUpdates)
const {
3853 if (IgnoreSPUpdates) {
3856 FrameReg = AArch64::SP;
3866 FrameReg = AArch64::SP;
3891 bool IsValid =
false;
3893 int ObjectIndex = 0;
3895 int GroupIndex = -1;
3897 bool ObjectFirst =
false;
3900 bool GroupFirst =
false;
3905 enum { AccessFPR = 1, AccessHazard = 2, AccessGPR = 4 };
3909 SmallVector<int, 8> CurrentMembers;
3910 int NextGroupIndex = 0;
3911 std::vector<FrameObject> &Objects;
3914 GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
3915 void AddMember(
int Index) { CurrentMembers.
push_back(Index); }
3916 void EndCurrentGroup() {
3917 if (CurrentMembers.
size() > 1) {
3922 for (
int Index : CurrentMembers) {
3923 Objects[
Index].GroupIndex = NextGroupIndex;
3929 CurrentMembers.clear();
3933bool FrameObjectCompare(
const FrameObject &
A,
const FrameObject &
B) {
3955 return std::make_tuple(!
A.IsValid,
A.Accesses,
A.ObjectFirst,
A.GroupFirst,
3956 A.GroupIndex,
A.ObjectIndex) <
3957 std::make_tuple(!
B.IsValid,
B.Accesses,
B.ObjectFirst,
B.GroupFirst,
3958 B.GroupIndex,
B.ObjectIndex);
3968 ObjectsToAllocate.
empty())
3973 for (
auto &Obj : ObjectsToAllocate) {
3974 FrameObjects[Obj].IsValid =
true;
3975 FrameObjects[Obj].ObjectIndex = Obj;
3980 GroupBuilder GB(FrameObjects);
3981 for (
auto &
MBB : MF) {
3982 for (
auto &
MI :
MBB) {
3983 if (
MI.isDebugInstr())
3988 if (FI && *FI >= 0 && *FI < (
int)FrameObjects.size()) {
3991 FrameObjects[*FI].Accesses |= FrameObject::AccessFPR;
3993 FrameObjects[*FI].Accesses |= FrameObject::AccessGPR;
3998 switch (
MI.getOpcode()) {
3999 case AArch64::STGloop:
4000 case AArch64::STZGloop:
4004 case AArch64::STZGi:
4005 case AArch64::ST2Gi:
4006 case AArch64::STZ2Gi:
4019 FrameObjects[FI].IsValid)
4027 GB.AddMember(TaggedFI);
4029 GB.EndCurrentGroup();
4032 GB.EndCurrentGroup();
4037 FrameObject::AccessHazard;
4039 for (
auto &Obj : FrameObjects)
4040 if (!Obj.Accesses ||
4041 Obj.Accesses == (FrameObject::AccessGPR | FrameObject::AccessFPR))
4042 Obj.Accesses = FrameObject::AccessGPR;
4051 FrameObjects[*TBPI].ObjectFirst =
true;
4052 FrameObjects[*TBPI].GroupFirst =
true;
4053 int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
4054 if (FirstGroupIndex >= 0)
4055 for (FrameObject &Object : FrameObjects)
4056 if (Object.GroupIndex == FirstGroupIndex)
4057 Object.GroupFirst =
true;
4063 for (
auto &Obj : FrameObjects) {
4067 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4071 dbgs() <<
"Final frame order:\n";
4072 for (
auto &Obj : FrameObjects) {
4075 dbgs() <<
" " << Obj.ObjectIndex <<
": group " << Obj.GroupIndex;
4076 if (Obj.ObjectFirst)
4077 dbgs() <<
", first";
4079 dbgs() <<
", group-first";
4090AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
4101 MF.
insert(MBBInsertPoint, LoopMBB);
4103 MF.
insert(MBBInsertPoint, ExitMBB);
4138 MBB.addSuccessor(LoopMBB);
4142 return ExitMBB->
begin();
4145void AArch64FrameLowering::inlineStackProbeFixed(
4150 const AArch64InstrInfo *
TII =
4152 AArch64FunctionInfo *AFI = MF.
getInfo<AArch64FunctionInfo>();
4157 int64_t ProbeSize = MF.
getInfo<AArch64FunctionInfo>()->getStackProbeSize();
4158 int64_t NumBlocks = FrameSize / ProbeSize;
4159 int64_t ResidualSize = FrameSize % ProbeSize;
4161 LLVM_DEBUG(
dbgs() <<
"Stack probing: total " << FrameSize <<
" bytes, "
4162 << NumBlocks <<
" blocks of " << ProbeSize
4163 <<
" bytes, plus " << ResidualSize <<
" bytes\n");
4168 for (
int i = 0; i < NumBlocks; ++i) {
4174 EmitAsyncCFI && !HasFP, CFAOffset);
4187 }
else if (NumBlocks != 0) {
4193 EmitAsyncCFI && !HasFP, CFAOffset);
4195 MBBI = inlineStackProbeLoopExactMultiple(
MBBI, ProbeSize, ScratchReg);
4197 if (EmitAsyncCFI && !HasFP) {
4200 .buildDefCFARegister(AArch64::SP);
4204 if (ResidualSize != 0) {
4210 EmitAsyncCFI && !HasFP, CFAOffset);
4231 SmallVector<MachineInstr *, 4> ToReplace;
4232 for (MachineInstr &
MI :
MBB)
4233 if (
MI.getOpcode() == AArch64::PROBED_STACKALLOC ||
4234 MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
4237 for (MachineInstr *
MI : ToReplace) {
4238 if (
MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
4239 Register ScratchReg =
MI->getOperand(0).getReg();
4240 int64_t FrameSize =
MI->getOperand(1).getImm();
4242 MI->getOperand(3).getImm());
4243 inlineStackProbeFixed(
MI->getIterator(), ScratchReg, FrameSize,
4246 assert(
MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
4247 "Stack probe pseudo-instruction expected");
4248 const AArch64InstrInfo *
TII =
4249 MI->getMF()->getSubtarget<AArch64Subtarget>().getInstrInfo();
4250 Register TargetReg =
MI->getOperand(0).getReg();
4251 (void)
TII->probedStackAlloc(
MI->getIterator(), TargetReg,
true);
4253 MI->eraseFromParent();
4273 return std::make_tuple(
start(),
Idx) <
4274 std::make_tuple(Rhs.
start(), Rhs.
Idx);
4304 << (
Offset.getFixed() < 0 ?
"" :
"+") <<
Offset.getFixed();
4305 if (
Offset.getScalable())
4306 OS << (
Offset.getScalable() < 0 ?
"" :
"+") <<
Offset.getScalable()
4317void AArch64FrameLowering::emitRemarks(
4320 auto *AFI = MF.
getInfo<AArch64FunctionInfo>();
4328 .stack_hazard_remark_size;
4330 if (HazardSize == 0)
4338 std::vector<StackAccess> StackAccesses(MFI.
getNumObjects());
4340 size_t NumFPLdSt = 0;
4341 size_t NumNonFPLdSt = 0;
4344 for (
const MachineBasicBlock &
MBB : MF) {
4345 for (
const MachineInstr &
MI :
MBB) {
4346 if (!
MI.mayLoadOrStore() ||
MI.getNumMemOperands() < 1)
4348 for (MachineMemOperand *MMO :
MI.memoperands()) {
4355 StackAccesses[ArrIdx].Idx = FrameIdx;
4356 StackAccesses[ArrIdx].Offset =
4367 StackAccesses[ArrIdx].AccessTypes |= RegTy;
4378 if (NumFPLdSt == 0 || NumNonFPLdSt == 0)
4389 if (StackAccesses.front().isMixed())
4390 MixedObjects.push_back(&StackAccesses.front());
4392 for (
auto It = StackAccesses.begin(),
End = std::prev(StackAccesses.end());
4394 const auto &
First = *It;
4395 const auto &Second = *(It + 1);
4397 if (Second.isMixed())
4398 MixedObjects.push_back(&Second);
4400 if ((
First.isSME() && Second.isCPU()) ||
4401 (
First.isCPU() && Second.isSME())) {
4403 if (Distance < HazardSize)
4408 auto EmitRemark = [&](llvm::StringRef Str) {
4410 auto R = MachineOptimizationRemarkAnalysis(
4411 "sme",
"StackHazard", MF.getFunction().getSubprogram(), &MF.front());
4412 return R <<
formatv(
"stack hazard in '{0}': ", MF.getName()).str() << Str;
4416 for (
const auto &
P : HazardPairs)
4417 EmitRemark(
formatv(
"{0} is too close to {1}", *
P.first, *
P.second).str());
4419 for (
const auto *Obj : MixedObjects)
4421 formatv(
"{0} accessed by both GP and FP instructions", *Obj).str());
static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB)
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
static RegState getPrologueDeath(MachineFunction &MF, unsigned Reg)
static bool produceCompactUnwindFrame(const AArch64FrameLowering &, MachineFunction &MF)
static bool invalidateWindowsRegisterPairing(bool SpillExtendedVolatile, int PairOffset, unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, const TargetRegisterInfo *TRI)
bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget, MachineFunction &MF)
static std::optional< int > getLdStFrameID(const MachineInstr &MI, const MachineFrameInfo &MFI)
static void orderZPRCalleeSavesForGroups(MachineFunction &MF, const TargetRegisterInfo *RegInfo, std::vector< CalleeSavedInfo > &CSI)
void computeCalleeSaveRegisterGroups(const AArch64FrameLowering &AFL, MachineFunction &MF, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegGroupInfo > &RegGroups, bool NeedsFrameRecord)
static bool invalidateRegisterPairing(bool SpillExtendedVolatile, int PairOffset, unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, const TargetRegisterInfo *TRI)
Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
static bool isLikelyToHaveSVEStack(const AArch64FrameLowering &AFL, const MachineFunction &MF)
static SVEStackSizes determineSVEStackSizes(MachineFunction &MF, AssignObjectOffsets AssignOffsets)
Process all the SVE stack objects and the SVE stack size and offsets for each object.
static bool isTargetWindows(const MachineFunction &MF)
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI, int &Min, int &Max)
returns true if there are any SVE callee saves.
static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE)
static unsigned getStackHazardSize(const MachineFunction &MF)
static bool isValidMemOpOffset(const AArch64InstrInfo *TII, unsigned Opcode, int Offset)
MCRegister findFreePredicateReg(BitVector &SavedRegs)
static bool isPPRAccess(const MachineInstr &MI)
static std::optional< int > getMMOFrameID(MachineMemOperand *MMO, const MachineFrameInfo &MFI)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file contains the declaration of the AArch64PrologueEmitter and AArch64EpilogueEmitter classes,...
static const int kSetTagLoopThreshold
const AArch64InstrInfo & TII
AMDGPU Reserve WWM Registers
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file contains the simple types necessary to represent the attributes associated with functions a...
#define CASE(ATTRNAME, AANAME,...)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
DXIL Forward Handle Accesses
Module.h This file contains the declarations for the Module class.
static std::string getTypeString(Type *T)
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
Register const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
MachineRegisterInfo & MRI
This file declares the machine register scavenger class.
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
This file defines the SmallVector class.
void emitEpilogue()
Emit the epilogue.
StackOffset getSVEStackSize(const MachineFunction &MF) const
Returns the size of the entire SVE stackframe (PPRs + ZPRs).
StackOffset getZPRStackSize(const MachineFunction &MF) const
Returns the size of the entire ZPR stackframe (calleesaves + spills).
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
bool enableFullCFIFixup(const MachineFunction &MF) const override
enableFullCFIFixup - Returns true if we may need to fix the unwind information such that it is accura...
StackOffset getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI) const override
getFrameIndexReferenceFromSP - This method returns the offset from the stack pointer to the slot of t...
bool enableCFIFixup(const MachineFunction &MF) const override
Returns true if we may need to fix the unwind information for the function.
StackOffset getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
friend class AArch64PrologueEmitter
bool hasFPImpl(const MachineFunction &MF) const override
hasFPImpl - Return true if the specified function should have a dedicated frame pointer register.
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
friend class AArch64EpilogueEmitter
void resetCFIToInitialState(MachineBasicBlock &MBB) const override
Emit CFI instructions that recreate the state of the unwind information upon function entry.
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
bool hasSVECalleeSavesAboveFrameRecord(const MachineFunction &MF) const
StackOffset resolveFrameOffsetReference(const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, TargetStackID::Value StackID, Register &FrameReg, bool PreferFP, bool ForSimm) const
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
bool isFPReserved(const MachineFunction &MF) const
Should the Frame Pointer be reserved for the current function?
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl< int > &ObjectsToAllocate) const override
Order the symbols in the local stack frame.
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
StackOffset getPPRStackSize(const MachineFunction &MF) const
Returns the size of the entire PPR stackframe (calleesaves + spills + hazard padding).
int64_t getArgumentStackToRestore(MachineFunction &MF, MachineBasicBlock &MBB) const
Returns how much of the incoming argument stack area (in bytes) we should clean up in an epilogue.
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
bool authenticatesReturnAddressEarly(MachineFunction &MF, MachineBasicBlock &MBB) const
Whether the return address is authenticated before the epilogue starts, in the return block MBB,...
bool enableWindowsCFIShrinkWrapping(const MachineFunction &MF) const override
Returns true if Windows CFI shrink wrapping should be enabled by default.
StackOffset getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, Register &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
StackOffset resolveFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP, bool ForSimm) const
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent's frame pointer...
bool requiresSaveVG(const MachineFunction &MF) const
void emitPacRetPlusLeafHardening(MachineFunction &MF) const
Harden the entire function with pac-ret.
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
unsigned getPPRCalleeSavedStackSize() const
void setHasStackFrame(bool s)
void setSwiftAsyncContextFrameIdx(int FI)
unsigned getTailCallReservedStack() const
unsigned getCalleeSavedStackSize(const MachineFrameInfo &MFI) const
void setCalleeSaveBaseToFrameRecordOffset(int Offset)
bool hasStackProbing() const
unsigned getArgumentStackToRestore() const
void setCalleeSaveStackHasFreeSpace(bool s)
int getCalleeSaveBaseToFrameRecordOffset() const
SignReturnAddress getSignReturnAddressCondition() const
bool hasStreamingModeChanges() const
void setPredicateRegForFillSpill(unsigned Reg)
bool hasWinCallToCalleePopFunction() const
int getStackHazardSlotIndex() const
void setCalleeSavedStackSize(unsigned Size)
void setSplitSVEObjects(bool s)
bool hasStackFrame() const
void setStackSizeSVE(uint64_t ZPR, uint64_t PPR)
std::optional< int > getTaggedBasePointerIndex() const
SMEAttrs getSMEFnAttrs() const
uint64_t getLocalStackSize() const
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
unsigned getVarArgsGPRSize() const
uint64_t getStackSizePPR() const
bool hasSwiftAsyncContext() const
bool hasStackHazardSlotIndex() const
void setStackHazardSlotIndex(int Index)
unsigned getZPRCalleeSavedStackSize() const
static bool shouldSignReturnAddress(SignReturnAddress Condition, bool IsLRSpilled)
void setStackHazardCSRSlotIndex(int Index)
unsigned getPredicateRegForFillSpill() const
void setSVECalleeSavedStackSize(unsigned ZPR, unsigned PPR)
bool hasCalculatedStackSizeSVE() const
uint64_t getStackSizeZPR() const
bool hasSVEStackSize() const
bool isStackHazardIncludedInCalleeSaveArea() const
unsigned getSVECalleeSavedStackSize() const
bool hasSplitSVEObjects() const
bool needsAsyncDwarfUnwindInfo(const MachineFunction &MF) const
bool hasCalleeSaveStackFreeSpace() const
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
void emitPrologue()
Emit the prologue.
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
const AArch64Options & getCLOpts() const
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
bool isTargetWindowsOrUEFI() const
UEFI follows the Windows ARM64 ABI and produces PE/COFF images, so most of the Windows specific ABI h...
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
bool test(unsigned Idx) const
Returns true if bit Idx is set.
BitVector & reset()
Reset all bits in the bitvector.
size_type count() const
Returns the number of bits which are set.
BitVector & set()
Set all bits in the bitvector.
iterator_range< const_set_bits_iterator > set_bits() const
size_type size() const
Returns the number of bits in this bitvector.
Helper class for creating CFI instructions and inserting them into MIR.
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Module * getParent()
Get the module that this global value is contained inside of...
A set of physical registers with utility functions to track liveness when walking backward/forward th...
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MachineInstr & instr_back()
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
reverse_iterator rbegin()
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
const AllocaInst * getObjectAllocation(int ObjectIdx) const
Return the underlying Alloca of the specified stack object if it exists.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
bool isCalleeSavedObjectIndex(int ObjectIdx) const
uint64_t getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
bool hasScalableStackID(int ObjectIdx) const
int getStackProtectorIndex() const
Return the index for the stack protector object.
LLVM_ABI uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isMaxCallFrameSizeComputed() const
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
LLVM_ABI int CreateSpillStackObject(uint64_t Size, Align Alignment, TargetStackID::Value StackID=TargetStackID::Default)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
unsigned getNumObjects() const
Return the number of objects.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
bool hasStackProtectorIndex() const
bool hasStackObjects() const
Return true if there are any stack objects in this function.
uint8_t getStackID(int ObjectIdx) const
unsigned getNumFixedObjects() const
Return the number of fixed objects.
void setIsCalleeSavedObjectIndex(int ObjectIdx, bool IsCalleeSaved)
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
int getObjectIndexBegin() const
Return the minimum frame object index.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
bool framePointerIsReserved() const
Returns true if the frame pointer must always either point to a new frame record or be un-modified in...
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
bool needsWinCFI() const
True if this function uses SEH unwind opcodes and needs an unwind table entry.
Function & getFunction()
Return the LLVM function that this machine code represents.
bool usesWindowsUnwindInfo() const
True if this function describes its unwind information with Windows SEH unwind opcodes (....
BasicBlockListType::iterator iterator
bool disableFramePointerElim() const
Returns true if frame pointer elimination should be disabled for this function.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
bool hasEHFunclets() const
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
void setFlags(unsigned flags)
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const Value * getValue() const
Return the base address of the memory access.
MachineOperand class - Representation of each machine instruction operand.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
LLVM_ABI bool isLiveIn(Register Reg) const
LLVM_ABI const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
LLVM_ABI bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
MutableArrayRef< T > drop_until(PredicateT Pred) const
Return a copy of *this with the first N elements not satisfying the given predicate removed.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool hasStreamingInterface() const
bool hasNonStreamingInterfaceAndBody() const
bool hasStreamingBody() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool enableCFIFixup(const MachineFunction &MF) const
Returns true if we may need to fix the unwind information for the function.
const Triple & getTargetTriple() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Triple - Helper class for working with autoconf configuration names.
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
const AArch64MCOptions & getCLOpts(const MCTargetOptions &Options)
const unsigned StackProbeMaxLoopUnroll
Maximum number of iterations to unroll for a constant size probing loop.
const unsigned StackProbeMaxUnprobedStack
Maximum allowed number of unprobed bytes above SP at an ABI boundary.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ PreserveNone
Used for runtime calls that preserves none general registers.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ C
The default llvm calling convention, compatible with C.
@ ScalablePredicateVector
NodeAddr< InstrNode * > Instr
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
void stable_sort(R &&Range)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
@ Unknown
Not known to have no common set bits.
RegState
Flags to represent properties of register accesses.
@ Define
Register definition.
@ LLVM_MARK_AS_BITMASK_ENUM
constexpr RegState getKillRegState(bool B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr RegState getDefRegState(bool B)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
auto reverse_conditionally(ContainerTy &&C, bool ShouldReverse)
Return a range that conditionally reverses C.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
@ End
Run the machine verifier once, at the end of the pipeline.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
MCRegisterClass TargetRegisterClass
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
bool operator<(const StackAccess &Rhs) const
void print(raw_ostream &OS) const
std::string getTypeString() const
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Pair of physical register and lane mask.
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
SmallVector< WinEHTryBlockMapEntry, 4 > TryBlockMap
SmallVector< WinEHHandlerType, 1 > HandlerArray