30#define DEBUG_TYPE "aarch64-stack-tagging-pre-ra"
38 "Unconditionally apply unchecked-ld-st optimization (even for large "
39 "stack frames, or in the presence of variable sized allocas)."),
44 "apply unchecked-ld-st when the target is definitely within range"),
49 cl::desc(
"Apply first slot optimization for stack tagging "
50 "(eliminate ADDG Rt, Rn, 0, 0)."));
70 bool mayUseUncheckedLoadStore();
71 void uncheckUsesOf(
unsigned TaggedReg,
int FI);
72 void uncheckLoadsAndStores();
73 std::optional<int> findFirstSlotCandidate();
77 return "AArch64 Stack Tagging PreRA";
87char AArch64StackTaggingPreRA::ID = 0;
90 "AArch64 Stack Tagging PreRA Pass",
false,
false)
95 return new AArch64StackTaggingPreRA();
100 case AArch64::LDRBBui:
101 case AArch64::LDRHHui:
102 case AArch64::LDRWui:
103 case AArch64::LDRXui:
105 case AArch64::LDRBui:
106 case AArch64::LDRHui:
107 case AArch64::LDRSui:
108 case AArch64::LDRDui:
109 case AArch64::LDRQui:
111 case AArch64::LDRSHWui:
112 case AArch64::LDRSHXui:
114 case AArch64::LDRSBWui:
115 case AArch64::LDRSBXui:
117 case AArch64::LDRSWui:
119 case AArch64::STRBBui:
120 case AArch64::STRHHui:
121 case AArch64::STRWui:
122 case AArch64::STRXui:
124 case AArch64::STRBui:
125 case AArch64::STRHui:
126 case AArch64::STRSui:
127 case AArch64::STRDui:
128 case AArch64::STRQui:
136 case AArch64::LDPSWi:
149bool AArch64StackTaggingPreRA::mayUseUncheckedLoadStore() {
166 unsigned FrameSize = 0;
169 bool EntireFrameReachableFromSP = FrameSize < 0xf00;
173void AArch64StackTaggingPreRA::uncheckUsesOf(
unsigned TaggedReg,
int FI) {
178 unsigned OpIdx =
TII->getLoadStoreImmIdx(UseI.getOpcode()) - 1;
179 if (UseI.getOperand(OpIdx).isReg() &&
180 UseI.getOperand(OpIdx).getReg() == TaggedReg) {
181 UseI.getOperand(OpIdx).ChangeToFrameIndex(FI);
184 }
else if (UseI.isCopy() && UseI.getOperand(0).getReg().isVirtual()) {
185 uncheckUsesOf(UseI.getOperand(0).getReg(), FI);
190void AArch64StackTaggingPreRA::uncheckLoadsAndStores() {
191 for (
auto *
I : ReTags) {
192 Register TaggedReg =
I->getOperand(0).getReg();
193 int FI =
I->getOperand(1).getIndex();
194 uncheckUsesOf(TaggedReg, FI);
202 SlotWithTag(
int FI,
int Tag) : FI(FI),
Tag(
Tag) {}
204 : FI(
MI.getOperand(1).getIndex()),
Tag(
MI.getOperand(4).getImm()) {}
219 static bool isEqual(
const SlotWithTag &
A,
const SlotWithTag &
B) {
236std::optional<int> AArch64StackTaggingPreRA::findFirstSlotCandidate() {
251 LLVM_DEBUG(
dbgs() <<
"AArch64StackTaggingPreRA::findFirstSlotCandidate\n");
256 SlotWithTag MaxScoreST{-1, -1};
258 for (
auto *
I : ReTags) {
263 Register RetagReg =
I->getOperand(0).getReg();
271 while (!WorkList.
empty()) {
273 for (
auto &UseI :
MRI->use_instructions(
UseReg)) {
274 unsigned Opcode = UseI.getOpcode();
275 if (Opcode == AArch64::STGi || Opcode == AArch64::ST2Gi ||
276 Opcode == AArch64::STZGi || Opcode == AArch64::STZ2Gi ||
277 Opcode == AArch64::STGPi || Opcode == AArch64::STGloop ||
278 Opcode == AArch64::STZGloop || Opcode == AArch64::STGloop_wback ||
279 Opcode == AArch64::STZGloop_wback)
282 Register DstReg = UseI.getOperand(0).getReg();
294 int TotalScore = RetagScore[
ST] += Score;
295 if (TotalScore > MaxScore ||
296 (TotalScore == MaxScore &&
ST.FI > MaxScoreST.FI)) {
297 MaxScore = TotalScore;
302 if (MaxScoreST.FI < 0)
306 if (MaxScoreST.Tag == 0)
307 return MaxScoreST.FI;
310 SlotWithTag SwapST{-1, -1};
311 for (
auto *
I : ReTags) {
322 for (
auto *&
I : ReTags) {
325 if (ST == MaxScoreST) {
327 }
else if (ST == SwapST) {
328 TagOp.
setImm(MaxScoreST.Tag);
331 return MaxScoreST.FI;
334bool AArch64StackTaggingPreRA::runOnMachineFunction(
MachineFunction &Func) {
346 LLVM_DEBUG(
dbgs() <<
"********** AArch64 Stack Tagging PreRA **********\n"
347 <<
"********** Function: " << MF->
getName() <<
'\n');
350 for (
auto &BB : *MF) {
352 if (
I.getOpcode() == AArch64::TAGPstack) {
353 ReTags.push_back(&
I);
354 int FI =
I.getOperand(1).getIndex();
357 assert(
I.getOperand(2).getImm() == 0);
364 for (
int FI : TaggedSlots)
370 if (mayUseUncheckedLoadStore())
371 uncheckLoadsAndStores();
376 std::optional<int> BaseSlot = findFirstSlotCandidate();
380 for (
auto *
I : ReTags) {
381 int FI =
I->getOperand(1).getIndex();
382 int Tag =
I->getOperand(4).getImm();
384 if (Tag == 0 && FI == BaseSlot) {
385 BuildMI(*
I->getParent(),
I, {},
TII->get(AArch64::COPY),
386 I->getOperand(0).getReg())
388 I->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
aarch64 stack tagging pre ra
static bool isSlotPreAllocated(MachineFrameInfo *MFI, int FI)
cl::opt< UncheckedLdStMode > ClUncheckedLdSt("stack-tagging-unchecked-ld-st", cl::Hidden, cl::init(UncheckedSafe), cl::desc("Unconditionally apply unchecked-ld-st optimization (even for large " "stack frames, or in the presence of variable sized allocas)."), cl::values(clEnumValN(UncheckedNever, "never", "never apply unchecked-ld-st"), clEnumValN(UncheckedSafe, "safe", "apply unchecked-ld-st when the target is definitely within range"), clEnumValN(UncheckedAlways, "always", "always apply unchecked-ld-st")))
static cl::opt< bool > ClFirstSlot("stack-tagging-first-slot-opt", cl::Hidden, cl::init(true), cl::desc("Apply first slot optimization for stack tagging " "(eliminate ADDG Rt, Rn, 0, 0)."))
static bool isUncheckedLoadOrStoreOpcode(unsigned Opcode)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
std::optional< std::vector< StOtherPiece > > Other
static Register UseReg(const MachineOperand &MO)
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
void setTaggedBasePointerIndex(int Index)
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
FunctionPass class - This class is used to implement most global optimizations.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
bool isObjectPreAllocated(int ObjectIdx) const
Return true if the object was pre-allocated into the local block.
@ SSPLK_None
Did not trigger a stack protector.
void setObjectSSPLayout(int ObjectIdx, SSPLayoutKind Kind)
bool getUseLocalStackAllocationBlock() const
Get whether the local allocation blob should be allocated together or let PEI allocate the locals in ...
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createAArch64StackTaggingPreRAPass()
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void initializeAArch64StackTaggingPreRAPass(PassRegistry &)
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
static bool isEqual(const SlotWithTag &A, const SlotWithTag &B)
static SlotWithTag getTombstoneKey()
static SlotWithTag getEmptyKey()
static unsigned getHashValue(const SlotWithTag &V)
An information struct used to provide DenseMap with the various necessary components for a given valu...