37#include "llvm/IR/IntrinsicsAArch64.h"
53#define DEBUG_TYPE "aarch64-stack-tagging"
57 cl::desc(
"merge stack variable initializers with tagging when possible"));
62 cl::desc(
"Use Stack Safety analysis results"));
74 cl::desc(
"How many lifetime ends to handle for a single alloca."),
89 "stack-tagging-record-stack-history",
90 cl::desc(
"Record stack frames with tagged allocations in a thread-local "
94 "storing into the stack ring buffer")),
101class InitializerBuilder {
117 std::map<uint64_t, Value *> Out;
120 InitializerBuilder(uint64_t Size,
const DataLayout *DL,
Value *BasePtr,
121 Function *SetTagFn, Function *SetTagZeroFn,
123 : Size(Size), DL(DL), BasePtr(BasePtr), SetTagFn(SetTagFn),
124 SetTagZeroFn(SetTagZeroFn), StgpFn(StgpFn) {}
126 bool addRange(uint64_t Start, uint64_t End, Instruction *Inst) {
131 if (
I != Ranges.end() && End >
I->Start) {
135 Ranges.insert(
I, {
Start, End, Inst});
139 bool addStore(uint64_t
Offset, StoreInst *SI,
const DataLayout *DL) {
140 int64_t StoreSize = DL->getTypeStoreSize(
SI->getOperand(0)->getType());
148 bool addMemSet(uint64_t
Offset, MemSetInst *MSI) {
158 void applyMemSet(
IRBuilder<> &IRB, int64_t Start, int64_t End,
166 uint64_t Cst = 0x0101010101010101UL;
169 Cst = (Cst >> LowBits) << LowBits;
170 int HighBits = End -
Offset < 8 ? (8 - (End -
Offset)) * 8 : 0;
172 Cst = (Cst << HighBits) >> HighBits;
174 ConstantInt::get(IRB.
getInt64Ty(), Cst *
V->getZExtValue());
200 void applyStore(
IRBuilder<> &IRB, int64_t Start, int64_t End,
201 Value *StoredValue) {
202 StoredValue = flatten(IRB, StoredValue);
204 Value *
V = sliceValue(IRB, StoredValue,
Offset - Start);
209 CurrentV = IRB.
CreateOr(CurrentV, V);
217 if (Ranges.empty()) {
218 emitUndef(IRB, 0, Size);
225 uint64_t LastOffset = 0;
228 auto I2 = Out.find(
Offset + 8);
229 if (I1 == Out.end() && I2 == Out.end())
233 emitZeroes(IRB, LastOffset,
Offset - LastOffset);
239 emitPair(IRB,
Offset, Store1, Store2);
245 if (LastOffset < Size)
246 emitZeroes(IRB, LastOffset, Size - LastOffset);
248 for (
const auto &R : Ranges) {
249 R.Inst->eraseFromParent();
282 if (
V->getType()->isIntegerTy())
287 Type *EltTy = VecTy->getElementType();
289 uint32_t EltSize = DL->getTypeSizeInBits(EltTy);
297 V, IRB.
getIntNTy(DL->getTypeStoreSize(
V->getType()) * 8));
302 const bool MergeInit;
303 const bool UseStackSafety;
308 AArch64StackTagging(
bool IsOptNone =
false)
314 void tagAlloca(AllocaInst *AI, Instruction *InsertBefore,
Value *
Ptr,
316 void untagAlloca(AllocaInst *AI, Instruction *InsertBefore, uint64_t
Size);
319 uint64_t
Size, InitializerBuilder &IB);
323 const MapVector<AllocaInst *, memtag::AllocaInfo> &Allocas,
324 const DominatorTree *DT);
327 StringRef getPassName()
const override {
return "AArch64 Stack Tagging"; }
332 const DataLayout *DL =
nullptr;
333 AAResults *AA =
nullptr;
334 const StackSafetyGlobalInfo *SSI =
nullptr;
336 void getAnalysisUsage(AnalysisUsage &AU)
const override {
339 AU.
addRequired<StackSafetyGlobalInfoWrapperPass>();
342 AU.
addRequired<OptimizationRemarkEmitterWrapperPass>();
348char AArch64StackTagging::ID = 0;
359 return new AArch64StackTagging(IsOptNone);
365 InitializerBuilder &IB) {
381 if (BI->mayWriteToMemory() || BI->mayReadFromMemory())
387 if (!NextStore->isSimple())
391 std::optional<int64_t>
Offset =
392 NextStore->getPointerOperand()->getPointerOffsetFrom(StartPtr, *
DL);
398 LastInst = NextStore;
409 std::optional<int64_t>
Offset =
422void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore,
425 F->getParent(), Intrinsic::aarch64_settag_zero);
427 Intrinsic::aarch64_stgp);
429 InitializerBuilder
IB(
Size,
DL,
Ptr, SetTagFunc, SetTagZeroFunc, StgpFunc);
432 if (MergeInit && !
F->hasOptNone() && LittleEndian &&
435 <<
", size = " <<
Size <<
"\n");
436 InsertBefore = collectInitializers(InsertBefore,
Ptr,
Size, IB);
443void AArch64StackTagging::untagAlloca(AllocaInst *AI, Instruction *InsertBefore,
450Instruction *AArch64StackTagging::insertBaseTaggedPointer(
452 const MapVector<AllocaInst *, memtag::AllocaInfo> &AllocasToInstrument,
453 const DominatorTree *DT) {
456 for (
auto &
I : AllocasToInstrument) {
457 const memtag::AllocaInfo &
Info =
I.second;
458 AllocaInst *AI =
Info.AI;
472 const Triple &TargetTriple =
M.getTargetTriple();
478 !AllocasToInstrument.empty()) {
479 constexpr int StackMteSlot = -3;
480 constexpr uint64_t TagMask = 0xFULL << 56;
484 auto *ThreadLong = IRB.
CreateLoad(IntptrTy, SlotPtr);
499bool AArch64StackTagging::runOnFunction(Function &Fn) {
504 SSI = &getAnalysis<StackSafetyGlobalInfoWrapperPass>().getResult();
508 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
509 OptimizationRemarkEmitter &ORE =
510 getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
512 memtag::StackInfoBuilder SIB(SSI,
DEBUG_TYPE);
515 memtag::StackInfo &SInfo = SIB.get();
520 std::unique_ptr<DominatorTree> DeleteDT;
521 DominatorTree *DT =
nullptr;
522 if (
auto *
P = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
523 DT = &
P->getDomTree();
526 DeleteDT = std::make_unique<DominatorTree>(*
F);
530 std::unique_ptr<PostDominatorTree> DeletePDT;
531 PostDominatorTree *PDT =
nullptr;
532 if (
auto *
P = getAnalysisIfAvailable<PostDominatorTreeWrapperPass>())
533 PDT = &
P->getPostDomTree();
535 if (PDT ==
nullptr) {
536 DeletePDT = std::make_unique<PostDominatorTree>(*
F);
537 PDT = DeletePDT.get();
540 std::unique_ptr<LoopInfo> DeleteLI;
541 LoopInfo *LI =
nullptr;
542 if (
auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>()) {
543 LI = &LIWP->getLoopInfo();
545 DeleteLI = std::make_unique<LoopInfo>(*DT);
550 Intrinsic::aarch64_settag);
555 unsigned int NextTag = 0;
557 memtag::AllocaInfo &
Info =
I.second;
561 AllocaInst *AI =
Info.AI;
562 unsigned int Tag = NextTag;
563 NextTag = (NextTag + 1) % 16;
570 if (
Info.AI->hasName())
573 Info.AI->replaceUsesWithIf(TagPCall, [&](
const Use &U) {
582 bool StandardLifetime =
586 if (StandardLifetime) {
587 IntrinsicInst *
Start =
Info.LifetimeStart[0];
588 uint64_t
Size = *
Info.AI->getAllocationSize(*
DL);
590 tagAlloca(AI,
Start->getNextNode(), TagPCall,
Size);
596 for (
auto *End :
Info.LifetimeEnd)
597 End->eraseFromParent();
600 uint64_t
Size = *
Info.AI->getAllocationSize(*
DL);
603 for (
auto *RI : SInfo.
RetVec) {
604 untagAlloca(AI, RI,
Size);
608 for (
auto *
II :
Info.LifetimeStart)
609 II->eraseFromParent();
610 for (
auto *
II :
Info.LifetimeEnd)
611 II->eraseFromParent();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static cl::opt< bool > ClMergeInit("stack-tagging-merge-init", cl::Hidden, cl::init(true), cl::desc("merge stack variable initializers with tagging when possible"))
StackTaggingRecordStackHistoryMode
static cl::opt< unsigned > ClMergeInitSizeLimit("stack-tagging-merge-init-size-limit", cl::init(272), cl::Hidden)
static cl::opt< unsigned > ClScanLimit("stack-tagging-merge-init-scan-limit", cl::init(40), cl::Hidden)
static cl::opt< bool > ClUseStackSafety("stack-tagging-use-stack-safety", cl::Hidden, cl::init(true), cl::desc("Use Stack Safety analysis results"))
static cl::opt< size_t > ClMaxLifetimes("stack-tagging-max-lifetimes-for-alloca", cl::Hidden, cl::init(3), cl::ReallyHidden, cl::desc("How many lifetime ends to handle for a single alloca."), cl::Optional)
static const Align kTagGranuleSize
static cl::opt< StackTaggingRecordStackHistoryMode > ClRecordStackHistory("stack-tagging-record-stack-history", cl::desc("Record stack frames with tagged allocations in a thread-local " "ring buffer"), cl::values(clEnumVal(none, "Do not record stack ring history"), clEnumVal(instr, "Insert instructions into the prologue for " "storing into the stack ring buffer")), cl::Hidden, cl::init(none))
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define clEnumVal(ENUMVAL, DESC)
This file contains constants used for implementing Dwarf debug support.
static bool runOnFunction(Function &F, bool PostInlining)
This header defines various interfaces for pass management in LLVM.
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static unsigned getNumElements(Type *Ty)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
ModRefInfo getModRefInfo(const Instruction *I, const std::optional< MemoryLocation > &OptLoc)
Check whether or not an instruction may read or write the optionally specified memory location.
AnalysisUsage & addRequired()
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
const Instruction & front() const
InstListType::iterator iterator
Instruction iterators...
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
LLVM_ABI Instruction * findNearestCommonDominator(Instruction *I1, Instruction *I2) const
Find the nearest instruction I that dominates both I1 and I2, in the sense that a result produced bef...
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionPass class - This class is used to implement most global optimizations.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Module * getParent()
Get the module that this global value is contained inside of...
Value * CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
BasicBlock::iterator GetInsertPoint() const
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getIntPtrTy(const DataLayout &DL, unsigned AddrSpace=0)
Fetch the type of an integer with size at least as big as that of a pointer in the given address spac...
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Value * getLength() const
Value * getDest() const
This is just like getRawDest, but it strips off any cast instructions (including addrspacecast) that ...
Representation for a specific memory location.
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
This pass performs the global (interprocedural) stack safety analysis (legacy pass manager).
bool isAndroidVersionLT(unsigned Major) const
bool isAndroid() const
Tests whether the target is Android.
LLVM_ABI bool isLittleEndian() const
Tests whether the target triple is little endian.
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
bool isPointerTy() const
True if this is an instance of PointerType.
void setOperand(unsigned i, Value *Val)
LLVM Value Representation.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI std::optional< int64_t > getPointerOffsetFrom(const Value *Other, const DataLayout &DL) const
If this ptr is provably equal to Other plus a constant offset, return that offset in bytes.
const ParentTy * getParent() const
@ C
The default llvm calling convention, compatible with C.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Value * getFP(IRBuilder<> &IRB)
bool isStandardLifetime(const SmallVectorImpl< IntrinsicInst * > &LifetimeStart, const SmallVectorImpl< IntrinsicInst * > &LifetimeEnd, const DominatorTree *DT, const LoopInfo *LI, size_t MaxLifetimes)
bool forAllReachableExits(const DominatorTree &DT, const PostDominatorTree &PDT, const LoopInfo &LI, const Instruction *Start, const SmallVectorImpl< IntrinsicInst * > &Ends, const SmallVectorImpl< Instruction * > &RetVec, llvm::function_ref< void(Instruction *)> Callback)
Value * getAndroidSlotPtr(IRBuilder<> &IRB, int Slot)
Value * incrementThreadLong(IRBuilder<> &IRB, Value *ThreadLong, unsigned int Inc)
void annotateDebugRecords(AllocaInfo &Info, unsigned int Tag)
void alignAndPadAlloca(memtag::AllocaInfo &Info, llvm::Align Align)
Value * getPC(const Triple &TargetTriple, IRBuilder<> &IRB)
NodeAddr< NodeBase * > Node
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionAddr VTableAddr Count
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
FunctionPass * createAArch64StackTaggingPass(bool IsOptNone)
bool isNoModRef(const ModRefInfo MRI)
This struct is a compact representation of a valid (non-zero power of two) alignment.
MapVector< AllocaInst *, AllocaInfo > AllocasToInstrument
SmallVector< Instruction *, 8 > RetVec