AArch64FrameLowering.cpp File Reference

#include "AArch64FrameLowering.h"
#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PrologueEpilogue.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "Utils/AArch64SMEAttributes.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Function.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <cassert>
#include <cstdint>
#include <iterator>
#include <optional>
#include <vector>

Go to the source code of this file.

Classes
struct	StackAccess

Macros
#define	DEBUG_TYPE   "frame-info"
#define	CASE(n)
#define	CASE(n)

Enumerations
enum class	AssignObjectOffsets { No , Yes }

Functions
static bool	produceCompactUnwindFrame (const AArch64FrameLowering &, MachineFunction &MF)
static SVEStackSizes	determineSVEStackSizes (MachineFunction &MF, AssignObjectOffsets AssignOffsets)
	Process all the SVE stack objects and the SVE stack size and offsets for each object.
static unsigned	getStackHazardSize (const MachineFunction &MF)
static bool	isLikelyToHaveSVEStack (const AArch64FrameLowering &AFL, const MachineFunction &MF)
static unsigned	estimateRSStackSizeLimit (MachineFunction &MF)
	Look at each instruction that references stack frames and return the stack size limit beyond which some of these instructions will require a scratch register during their expansion later.
static MCRegister	getRegisterOrZero (MCRegister Reg, bool HasSVE)
static void	getLiveRegsForEntryMBB (LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB)
static bool	isTargetWindows (const MachineFunction &MF)
static unsigned	getPrologueDeath (MachineFunction &MF, unsigned Reg)
static bool	invalidateWindowsRegisterPairing (unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool IsFirst, const TargetRegisterInfo *TRI)
static bool	invalidateRegisterPairing (unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, bool IsFirst, const TargetRegisterInfo *TRI)
	Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
unsigned	findFreePredicateReg (BitVector &SavedRegs)
bool	enableMultiVectorSpillFill (const AArch64Subtarget &Subtarget, MachineFunction &MF)
void	computeCalleeSaveRegisterPairs (const AArch64FrameLowering &AFL, MachineFunction &MF, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool NeedsFrameRecord)
static std::optional< int >	getMMOFrameID (MachineMemOperand *MMO, const MachineFrameInfo &MFI)
static std::optional< int >	getLdStFrameID (const MachineInstr &MI, const MachineFrameInfo &MFI)
static bool	isPPRAccess (const MachineInstr &MI)
static bool	getSVECalleeSaveSlotRange (const MachineFrameInfo &MFI, int &Min, int &Max)
	returns true if there are any SVE callee saves.
static raw_ostream &	operator<< (raw_ostream &OS, const StackAccess &SA)

Variables
static cl::opt< bool >	EnableRedZone ("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
static cl::opt< bool >	StackTaggingMergeSetTag ("stack-tagging-merge-settag", cl::desc("merge settag instruction in function epilog"), cl::init(true), cl::Hidden)
static cl::opt< bool >	OrderFrameObjects ("aarch64-order-frame-objects", cl::desc("sort stack allocations"), cl::init(true), cl::Hidden)
static cl::opt< bool >	SplitSVEObjects ("aarch64-split-sve-objects", cl::desc("Split allocation of ZPR & PPR objects"), cl::init(true), cl::Hidden)
cl::opt< bool >	EnableHomogeneousPrologEpilog ("homogeneous-prolog-epilog", cl::Hidden, cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)"))
static cl::opt< unsigned >	StackHazardRemarkSize ("aarch64-stack-hazard-remark-size", cl::init(0), cl::Hidden)
static cl::opt< bool >	StackHazardInNonStreaming ("aarch64-stack-hazard-in-non-streaming", cl::init(false), cl::Hidden)
static cl::opt< bool >	DisableMultiVectorSpillFill ("aarch64-disable-multivector-spill-fill", cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(false), cl::Hidden)
static const unsigned	DefaultSafeSPDisplacement = 255
	This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using a separate calculation and a temp register).

Macro Definition Documentation

CASE [1/2]

#define CASE

(

n

)

Value:

  case AArch64::W##n:                                                          \
  case AArch64::X##n:                                                          \
    return AArch64::X##n

CASE [2/2]

#define CASE

(

n

)

Value:

  case AArch64::B##n:                                                          \
  case AArch64::H##n:                                                          \
  case AArch64::S##n:                                                          \
  case AArch64::D##n:                                                          \
  case AArch64::Q##n:                                                          \
    return HasSVE ? AArch64::Z##n : AArch64::Q##n

DEBUG_TYPE

#define DEBUG_TYPE   "frame-info"

Definition at line 267 of file AArch64FrameLowering.cpp.

Enumeration Type Documentation

AssignObjectOffsets

enum class AssignObjectOffsets

strong

Enumerator
No
Yes

Definition at line 337 of file AArch64FrameLowering.cpp.

Function Documentation

computeCalleeSaveRegisterPairs()

void computeCalleeSaveRegisterPairs	(	const AArch64FrameLowering &	AFL,
		MachineFunction &	MF,
		ArrayRef< CalleeSavedInfo >	CSI,
		const TargetRegisterInfo *	TRI,
		SmallVectorImpl< RegPairInfo > &	RegPairs,
		bool	NeedsFrameRecord )

Definition at line 1636 of file AArch64FrameLowering.cpp.

References llvm::alignTo(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), contains(), llvm::Count, llvm::CallingConv::CXX_FAST_TLS, llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::AArch64FunctionInfo::getCalleeSavedStackSize(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::MachineFrameInfo::getObjectAlign(), llvm::AArch64FunctionInfo::getPPRCalleeSavedStackSize(), llvm::AArch64FunctionInfo::getPredicateRegForFillSpill(), getStackHazardSize(), llvm::AArch64FunctionInfo::getSVECalleeSavedStackSize(), llvm::AArch64FunctionInfo::getZPRCalleeSavedStackSize(), llvm::AArch64FunctionInfo::hasCalleeSaveStackFreeSpace(), llvm::AArch64FunctionInfo::hasSplitSVEObjects(), llvm::AArch64FunctionInfo::hasStackHazardSlotIndex(), llvm::AArch64FunctionInfo::hasSwiftAsyncContext(), invalidateRegisterPairing(), invalidateWindowsRegisterPairing(), llvm::AArch64InstrInfo::isFpOrNEON(), isTargetWindows(), llvm_unreachable, llvm::AArch64FrameLowering::needsWinCFI(), llvm::Offset, llvm::CallingConv::PreserveAll, llvm::CallingConv::PreserveMost, produceCompactUnwindFrame(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AArch64FunctionInfo::setCalleeSaveBaseToFrameRecordOffset(), llvm::MachineFrameInfo::setObjectAlignment(), llvm::ArrayRef< T >::size(), TRI, and llvm::CallingConv::Win64.

Referenced by llvm::AArch64FrameLowering::restoreCalleeSavedRegisters(), and llvm::AArch64FrameLowering::spillCalleeSavedRegisters().

determineSVEStackSizes()

SVEStackSizes determineSVEStackSizes ( MachineFunction & MF, AssignObjectOffsets AssignOffsets )

static

Process all the SVE stack objects and the SVE stack size and offsets for each object.

If AssignOffsets is "Yes", the offsets get assigned (and SVE stack sizes set). Returns the size of the SVE stack.

Definition at line 2843 of file AArch64FrameLowering.cpp.

References llvm::alignTo(), assert(), llvm::dbgs(), E(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getInfo(), llvm::MachineFrameInfo::getObjectAlign(), llvm::MachineFrameInfo::getObjectIndexBegin(), llvm::MachineFrameInfo::getObjectIndexEnd(), llvm::MachineFrameInfo::getObjectSize(), llvm::MachineFrameInfo::getStackID(), llvm::MachineFrameInfo::getStackProtectorIndex(), getSVECalleeSaveSlotRange(), llvm::MachineFrameInfo::hasScalableStackID(), llvm::AArch64FunctionInfo::hasSplitSVEObjects(), llvm::MachineFrameInfo::hasStackProtectorIndex(), I, llvm::MachineFrameInfo::isDeadObjectIndex(), LLVM_DEBUG, llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::report_fatal_error(), llvm::TargetStackID::ScalablePredicateVector, llvm::TargetStackID::ScalableVector, llvm::MachineFrameInfo::setObjectOffset(), llvm::AArch64FunctionInfo::setStackSizeSVE(), and Yes.

Referenced by llvm::AArch64FrameLowering::determineCalleeSaves(), and llvm::AArch64FrameLowering::processFunctionBeforeFrameFinalized().

enableMultiVectorSpillFill()

bool enableMultiVectorSpillFill	(	const AArch64Subtarget &	Subtarget,
		MachineFunction &	MF )

Definition at line 1619 of file AArch64FrameLowering.cpp.

References DisableMultiVectorSpillFill, llvm::MachineFunction::getInfo(), llvm::SMEAttrs::hasStreamingBody(), llvm::SMEAttrs::hasStreamingInterface(), and llvm::AArch64Subtarget::isStreaming().

Referenced by llvm::AArch64FrameLowering::determineCalleeSaves(), llvm::AArch64FrameLowering::restoreCalleeSavedRegisters(), and llvm::AArch64FrameLowering::spillCalleeSavedRegisters().

estimateRSStackSizeLimit()

unsigned estimateRSStackSizeLimit ( MachineFunction & MF )

static

Look at each instruction that references stack frames and return the stack size limit beyond which some of these instructions will require a scratch register during their expansion later.

Definition at line 452 of file AArch64FrameLowering.cpp.

References llvm::AArch64FrameOffsetCannotUpdate, DefaultSafeSPDisplacement, llvm::isAArch64FrameOffsetLegal(), MBB, MI, and llvm::Offset.

Referenced by llvm::AArch64FrameLowering::determineCalleeSaves(), llvm::ARMFrameLowering::determineCalleeSaves(), and llvm::CSKYFrameLowering::determineCalleeSaves().

findFreePredicateReg()

unsigned findFreePredicateReg

(

BitVector &

SavedRegs

)

Definition at line 1608 of file AArch64FrameLowering.cpp.

References llvm::BitVector::test().

Referenced by llvm::AArch64FrameLowering::determineCalleeSaves().

getLdStFrameID()

std::optional< int > getLdStFrameID ( const MachineInstr & MI, const MachineFrameInfo & MFI )

static

Definition at line 2271 of file AArch64FrameLowering.cpp.

References getMMOFrameID(), and MI.

Referenced by llvm::AArch64FrameLowering::orderFrameObjects().

getLiveRegsForEntryMBB()

void getLiveRegsForEntryMBB ( LivePhysRegs & LiveRegs, const MachineBasicBlock & MBB )

static

Definition at line 878 of file AArch64FrameLowering.cpp.

References llvm::MachineRegisterInfo::getCalleeSavedRegs(), llvm::MachineFunction::getRegInfo(), and MBB.

Referenced by llvm::AArch64FrameLowering::canUseAsPrologue().

getMMOFrameID()

std::optional< int > getMMOFrameID ( MachineMemOperand * MMO, const MachineFrameInfo & MFI )

static

Definition at line 2251 of file AArch64FrameLowering.cpp.

References llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::MachineFrameInfo::getObjectAllocation(), llvm::MachineFrameInfo::getObjectIndexBegin(), llvm::MachineFrameInfo::getObjectIndexEnd(), llvm::MachineMemOperand::getPseudoValue(), llvm::getUnderlyingObject(), and llvm::MachineMemOperand::getValue().

Referenced by getLdStFrameID().

getPrologueDeath()

unsigned getPrologueDeath ( MachineFunction & MF, unsigned Reg )

static

Definition at line 1520 of file AArch64FrameLowering.cpp.

References llvm::getKillRegState(), llvm::MachineFunction::getRegInfo(), llvm::MachineRegisterInfo::isLiveIn(), and Reg.

Referenced by llvm::AArch64FrameLowering::spillCalleeSavedRegisters().

getRegisterOrZero()

MCRegister getRegisterOrZero ( MCRegister Reg, bool HasSVE )

static

Definition at line 740 of file AArch64FrameLowering.cpp.

References CASE, and Reg.

getStackHazardSize()

unsigned getStackHazardSize ( const MachineFunction & MF )

static

Definition at line 344 of file AArch64FrameLowering.cpp.

References llvm::MachineFunction::getSubtarget().

Referenced by llvm::AArch64FrameLowering::assignCalleeSavedSpillSlots(), computeCalleeSaveRegisterPairs(), llvm::AArch64FrameLowering::determineCalleeSaves(), and llvm::AArch64FrameLowering::getPPRStackSize().

getSVECalleeSaveSlotRange()

bool getSVECalleeSaveSlotRange ( const MachineFrameInfo & MFI, int & Min, int & Max )

static

returns true if there are any SVE callee saves.

Definition at line 2821 of file AArch64FrameLowering.cpp.

References assert(), contains(), llvm::MachineFrameInfo::getCalleeSavedInfo(), and llvm::MachineFrameInfo::isCalleeSavedInfoValid().

Referenced by determineSVEStackSizes().

invalidateRegisterPairing()

bool invalidateRegisterPairing ( unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, bool IsFirst, const TargetRegisterInfo * TRI )

static

Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.

WindowsCFI requires that only consecutive registers can be paired. LR and FP need to be allocated together when the frame needs to save the frame-record. This means any other register pairing with LR is invalid.

Definition at line 1573 of file AArch64FrameLowering.cpp.

References invalidateWindowsRegisterPairing(), and TRI.

Referenced by computeCalleeSaveRegisterPairs().

invalidateWindowsRegisterPairing()

bool invalidateWindowsRegisterPairing ( unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool IsFirst, const TargetRegisterInfo * TRI )

static

Definition at line 1542 of file AArch64FrameLowering.cpp.

References TRI.

Referenced by computeCalleeSaveRegisterPairs(), and invalidateRegisterPairing().

isLikelyToHaveSVEStack()

bool isLikelyToHaveSVEStack ( const AArch64FrameLowering & AFL, const MachineFunction & MF )

static

Definition at line 368 of file AArch64FrameLowering.cpp.

References llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getInfo(), llvm::MachineFrameInfo::getObjectIndexBegin(), llvm::MachineFrameInfo::getObjectIndexEnd(), llvm::AArch64FrameLowering::getSVEStackSize(), llvm::AArch64FunctionInfo::hasCalculatedStackSizeSVE(), llvm::MachineFrameInfo::hasScalableStackID(), and llvm::AArch64FunctionInfo::isSVECC().

isPPRAccess()

bool isPPRAccess ( const MachineInstr & MI )

static

Definition at line 2280 of file AArch64FrameLowering.cpp.

References MI.

isTargetWindows()

bool isTargetWindows ( const MachineFunction & MF )

static

Definition at line 1156 of file AArch64FrameLowering.cpp.

References llvm::MachineFunction::getSubtarget(), and isTargetWindows().

Referenced by llvm::AArch64FrameLowering::assignCalleeSavedSpillSlots(), computeCalleeSaveRegisterPairs(), llvm::AArch64FrameLowering::getFrameIndexReferenceFromSP(), isTargetWindows(), and llvm::AArch64FrameLowering::resolveFrameOffsetReference().

operator<<()

raw_ostream & operator<< ( raw_ostream & OS, const StackAccess & SA )

inlinestatic

Definition at line 3908 of file AArch64FrameLowering.cpp.

References StackAccess::print().

produceCompactUnwindFrame()

bool produceCompactUnwindFrame ( const AArch64FrameLowering & AFL, MachineFunction & MF )

static

Definition at line 1530 of file AArch64FrameLowering.cpp.

References llvm::Function::getAttributes(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::MachineFunction::getSubtarget(), llvm::AArch64Subtarget::getTargetLowering(), llvm::AArch64FunctionInfo::isSVECC(), llvm::AArch64Subtarget::isTargetMachO(), llvm::AArch64FrameLowering::requiresSaveVG(), and llvm::CallingConv::SwiftTail.

Referenced by computeCalleeSaveRegisterPairs(), and llvm::AArch64FrameLowering::determineCalleeSaves().

Variable Documentation

DefaultSafeSPDisplacement

const unsigned DefaultSafeSPDisplacement = 255

static

This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using a separate calculation and a temp register).

Note that the exception here are vector stores/loads which cannot encode any displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).

Definition at line 447 of file AArch64FrameLowering.cpp.

Referenced by estimateRSStackSizeLimit(), and llvm::AArch64FrameLowering::hasFPImpl().

DisableMultiVectorSpillFill

cl::opt< bool > DisableMultiVectorSpillFill("aarch64-disable-multivector-spill-fill", cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(false), cl::Hidden) ( "aarch64-disable-multivector-spill-fill" , cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1") , cl::init(false) , cl::Hidden  )

static

Referenced by enableMultiVectorSpillFill().

EnableHomogeneousPrologEpilog

cl::opt< bool > EnableHomogeneousPrologEpilog("homogeneous-prolog-epilog", cl::Hidden, cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)"))	(	"homogeneous-prolog-epilog"	,
		cl::Hidden	,
		cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)")	)

EnableRedZone

cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden) ( "aarch64-redzone" , cl::desc("enable use of redzone on AArch64") , cl::init(false) , cl::Hidden  )

static

Referenced by llvm::AArch64FrameLowering::canUseRedZone().

OrderFrameObjects

cl::opt< bool > OrderFrameObjects("aarch64-order-frame-objects", cl::desc("sort stack allocations"), cl::init(true), cl::Hidden) ( "aarch64-order-frame-objects" , cl::desc("sort stack allocations") , cl::init(true) , cl::Hidden  )

static

Referenced by llvm::AArch64FrameLowering::orderFrameObjects().

SplitSVEObjects

cl::opt< bool > SplitSVEObjects("aarch64-split-sve-objects", cl::desc("Split allocation of ZPR & PPR objects"), cl::init(true), cl::Hidden) ( "aarch64-split-sve-objects" , cl::desc("Split allocation of ZPR & PPR objects") , cl::init(true) , cl::Hidden  )

static

StackHazardInNonStreaming

cl::opt< bool > StackHazardInNonStreaming("aarch64-stack-hazard-in-non-streaming", cl::init(false), cl::Hidden) ( "aarch64-stack-hazard-in-non-streaming" , cl::init(false) , cl::Hidden  )

static

StackHazardRemarkSize

cl::opt< unsigned > StackHazardRemarkSize("aarch64-stack-hazard-remark-size", cl::init(0), cl::Hidden) ( "aarch64-stack-hazard-remark-size" , cl::init(0) , cl::Hidden  )

static

StackTaggingMergeSetTag

cl::opt< bool > StackTaggingMergeSetTag("stack-tagging-merge-settag", cl::desc("merge settag instruction in function epilog"), cl::init(true), cl::Hidden) ( "stack-tagging-merge-settag" , cl::desc("merge settag instruction in function epilog") , cl::init(true) , cl::Hidden  )

static

Referenced by llvm::AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced().