34#define DEBUG_TYPE "aarch64-stack-tagging-pre-ra"
42 "Unconditionally apply unchecked-ld-st optimization (even for large "
43 "stack frames, or in the presence of variable sized allocas)."),
48 "apply unchecked-ld-st when the target is definitely within range"),
53 cl::desc(
"Apply first slot optimization for stack tagging "
54 "(eliminate ADDG Rt, Rn, 0, 0)."));
74 bool mayUseUncheckedLoadStore();
75 void uncheckUsesOf(
unsigned TaggedReg,
int FI);
76 void uncheckLoadsAndStores();
77 std::optional<int> findFirstSlotCandidate();
81 return "AArch64 Stack Tagging PreRA";
91char AArch64StackTaggingPreRA::ID = 0;
94 "AArch64 Stack Tagging PreRA Pass",
false,
false)
99 return new AArch64StackTaggingPreRA();
104 case AArch64::LDRBBui:
105 case AArch64::LDRHHui:
106 case AArch64::LDRWui:
107 case AArch64::LDRXui:
109 case AArch64::LDRBui:
110 case AArch64::LDRHui:
111 case AArch64::LDRSui:
112 case AArch64::LDRDui:
113 case AArch64::LDRQui:
115 case AArch64::LDRSHWui:
116 case AArch64::LDRSHXui:
118 case AArch64::LDRSBWui:
119 case AArch64::LDRSBXui:
121 case AArch64::LDRSWui:
123 case AArch64::STRBBui:
124 case AArch64::STRHHui:
125 case AArch64::STRWui:
126 case AArch64::STRXui:
128 case AArch64::STRBui:
129 case AArch64::STRHui:
130 case AArch64::STRSui:
131 case AArch64::STRDui:
132 case AArch64::STRQui:
140 case AArch64::LDPSWi:
153bool AArch64StackTaggingPreRA::mayUseUncheckedLoadStore() {
170 unsigned FrameSize = 0;
173 bool EntireFrameReachableFromSP = FrameSize < 0xf00;
177void AArch64StackTaggingPreRA::uncheckUsesOf(
unsigned TaggedReg,
int FI) {
182 unsigned OpIdx =
TII->getLoadStoreImmIdx(UseI.getOpcode()) - 1;
183 if (UseI.getOperand(OpIdx).isReg() &&
184 UseI.getOperand(OpIdx).getReg() == TaggedReg) {
185 UseI.getOperand(OpIdx).ChangeToFrameIndex(FI);
188 }
else if (UseI.isCopy() && UseI.getOperand(0).getReg().isVirtual()) {
189 uncheckUsesOf(UseI.getOperand(0).getReg(), FI);
194void AArch64StackTaggingPreRA::uncheckLoadsAndStores() {
195 for (
auto *
I : ReTags) {
196 Register TaggedReg =
I->getOperand(0).getReg();
197 int FI =
I->getOperand(1).getIndex();
198 uncheckUsesOf(TaggedReg, FI);
206 SlotWithTag(
int FI,
int Tag) : FI(FI),
Tag(
Tag) {}
208 : FI(
MI.getOperand(1).getIndex()),
Tag(
MI.getOperand(4).getImm()) {}
223 static bool isEqual(
const SlotWithTag &
A,
const SlotWithTag &
B) {
240std::optional<int> AArch64StackTaggingPreRA::findFirstSlotCandidate() {
255 LLVM_DEBUG(
dbgs() <<
"AArch64StackTaggingPreRA::findFirstSlotCandidate\n");
260 SlotWithTag MaxScoreST{-1, -1};
262 for (
auto *
I : ReTags) {
267 Register RetagReg =
I->getOperand(0).getReg();
275 while (!WorkList.
empty()) {
277 for (
auto &UseI :
MRI->use_instructions(
UseReg)) {
278 unsigned Opcode = UseI.getOpcode();
279 if (Opcode == AArch64::STGi || Opcode == AArch64::ST2Gi ||
280 Opcode == AArch64::STZGi || Opcode == AArch64::STZ2Gi ||
281 Opcode == AArch64::STGPi || Opcode == AArch64::STGloop ||
282 Opcode == AArch64::STZGloop || Opcode == AArch64::STGloop_wback ||
283 Opcode == AArch64::STZGloop_wback)
286 Register DstReg = UseI.getOperand(0).getReg();
298 int TotalScore = RetagScore[
ST] += Score;
299 if (TotalScore > MaxScore ||
300 (TotalScore == MaxScore &&
ST.FI > MaxScoreST.FI)) {
301 MaxScore = TotalScore;
306 if (MaxScoreST.FI < 0)
310 if (MaxScoreST.Tag == 0)
311 return MaxScoreST.FI;
314 SlotWithTag SwapST{-1, -1};
315 for (
auto *
I : ReTags) {
326 for (
auto *&
I : ReTags) {
329 if (ST == MaxScoreST) {
331 }
else if (ST == SwapST) {
332 TagOp.
setImm(MaxScoreST.Tag);
335 return MaxScoreST.FI;
338bool AArch64StackTaggingPreRA::runOnMachineFunction(
MachineFunction &Func) {
350 LLVM_DEBUG(
dbgs() <<
"********** AArch64 Stack Tagging PreRA **********\n"
351 <<
"********** Function: " << MF->
getName() <<
'\n');
354 for (
auto &BB : *MF) {
356 if (
I.getOpcode() == AArch64::TAGPstack) {
357 ReTags.push_back(&
I);
358 int FI =
I.getOperand(1).getIndex();
361 assert(
I.getOperand(2).getImm() == 0);
368 for (
int FI : TaggedSlots)
374 if (mayUseUncheckedLoadStore())
375 uncheckLoadsAndStores();
380 std::optional<int> BaseSlot = findFirstSlotCandidate();
384 for (
auto *
I : ReTags) {
385 int FI =
I->getOperand(1).getIndex();
386 int Tag =
I->getOperand(4).getImm();
388 if (
Tag == 0 && FI == BaseSlot) {
389 BuildMI(*
I->getParent(),
I, {},
TII->get(AArch64::COPY),
390 I->getOperand(0).getReg())
392 I->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
aarch64 stack tagging pre ra
static bool isSlotPreAllocated(MachineFrameInfo *MFI, int FI)
cl::opt< UncheckedLdStMode > ClUncheckedLdSt("stack-tagging-unchecked-ld-st", cl::Hidden, cl::init(UncheckedSafe), cl::desc("Unconditionally apply unchecked-ld-st optimization (even for large " "stack frames, or in the presence of variable sized allocas)."), cl::values(clEnumValN(UncheckedNever, "never", "never apply unchecked-ld-st"), clEnumValN(UncheckedSafe, "safe", "apply unchecked-ld-st when the target is definitely within range"), clEnumValN(UncheckedAlways, "always", "always apply unchecked-ld-st")))
static cl::opt< bool > ClFirstSlot("stack-tagging-first-slot-opt", cl::Hidden, cl::init(true), cl::desc("Apply first slot optimization for stack tagging " "(eliminate ADDG Rt, Rn, 0, 0)."))
static bool isUncheckedLoadOrStoreOpcode(unsigned Opcode)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
std::optional< std::vector< StOtherPiece > > Other
static Register UseReg(const MachineOperand &MO)
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
void setTaggedBasePointerIndex(int Index)
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
FunctionPass class - This class is used to implement most global optimizations.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
bool isObjectPreAllocated(int ObjectIdx) const
Return true if the object was pre-allocated into the local block.
@ SSPLK_None
Did not trigger a stack protector.
void setObjectSSPLayout(int ObjectIdx, SSPLayoutKind Kind)
bool getUseLocalStackAllocationBlock() const
Get whether the local allocation blob should be allocated together or let PEI allocate the locals in ...
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createAArch64StackTaggingPreRAPass()
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void initializeAArch64StackTaggingPreRAPass(PassRegistry &)
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
static bool isEqual(const SlotWithTag &A, const SlotWithTag &B)
static SlotWithTag getTombstoneKey()
static SlotWithTag getEmptyKey()
static unsigned getHashValue(const SlotWithTag &V)
An information struct used to provide DenseMap with the various necessary components for a given valu...