43#define DEBUG_TYPE "memprof"
59 "__memprof_version_mismatch_check_v";
62 "__memprof_shadow_memory_dynamic_address";
69 "memprof-guard-against-version-mismatch",
75 cl::desc(
"instrument read instructions"),
84 "memprof-instrument-atomics",
89 "memprof-use-callbacks",
90 cl::desc(
"Use callbacks instead of inline instrumentation sequences."),
95 cl::desc(
"Prefix for memory access callbacks"),
103 cl::desc(
"scale of memprof shadow mapping"),
108 cl::desc(
"granularity of memprof shadow mapping"),
112 cl::desc(
"Instrument scalar stack variables"),
129STATISTIC(NumInstrumentedReads,
"Number of instrumented reads");
130STATISTIC(NumInstrumentedWrites,
"Number of instrumented writes");
131STATISTIC(NumSkippedStackReads,
"Number of non-instrumented stack reads");
132STATISTIC(NumSkippedStackWrites,
"Number of non-instrumented stack writes");
138struct ShadowMapping {
142 Mask = ~(Granularity - 1);
155struct InterestingMemoryAccess {
160 Value *MaybeMask =
nullptr;
167 C = &(
M.getContext());
168 LongSize =
M.getDataLayout().getPointerSizeInBits();
175 std::optional<InterestingMemoryAccess>
179 InterestingMemoryAccess &Access);
188 bool maybeInsertMemProfInitAtFunctionEntry(
Function &
F);
189 bool insertDynamicShadowAtFunctionEntry(
Function &
F);
192 void initializeCallbacks(
Module &M);
197 ShadowMapping Mapping;
204 Value *DynamicShadowOffset =
nullptr;
207class ModuleMemProfiler {
209 ModuleMemProfiler(
Module &M) { TargetTriple =
Triple(
M.getTargetTriple()); }
211 bool instrumentModule(
Module &);
215 ShadowMapping Mapping;
216 Function *MemProfCtorFunction =
nullptr;
226 MemProfiler Profiler(M);
227 if (Profiler.instrumentFunction(
F))
236 ModuleMemProfiler Profiler(M);
237 if (Profiler.instrumentModule(M))
244 Shadow = IRB.
CreateAnd(Shadow, Mapping.Mask);
245 Shadow = IRB.
CreateLShr(Shadow, Mapping.Scale);
247 assert(DynamicShadowOffset);
248 return IRB.
CreateAdd(Shadow, DynamicShadowOffset);
254 if (isa<MemTransferInst>(
MI)) {
256 isa<MemMoveInst>(
MI) ? MemProfMemmove : MemProfMemcpy,
260 }
else if (isa<MemSetInst>(
MI)) {
267 MI->eraseFromParent();
270std::optional<InterestingMemoryAccess>
271MemProfiler::isInterestingMemoryAccess(
Instruction *
I)
const {
273 if (DynamicShadowOffset ==
I)
276 InterestingMemoryAccess Access;
278 if (
LoadInst *LI = dyn_cast<LoadInst>(
I)) {
281 Access.IsWrite =
false;
282 Access.AccessTy = LI->getType();
283 Access.Addr = LI->getPointerOperand();
284 }
else if (
StoreInst *SI = dyn_cast<StoreInst>(
I)) {
287 Access.IsWrite =
true;
288 Access.AccessTy =
SI->getValueOperand()->getType();
289 Access.Addr =
SI->getPointerOperand();
293 Access.IsWrite =
true;
294 Access.AccessTy = RMW->getValOperand()->getType();
295 Access.Addr = RMW->getPointerOperand();
299 Access.IsWrite =
true;
300 Access.AccessTy = XCHG->getCompareOperand()->getType();
301 Access.Addr = XCHG->getPointerOperand();
302 }
else if (
auto *CI = dyn_cast<CallInst>(
I)) {
303 auto *
F = CI->getCalledFunction();
304 if (
F && (
F->getIntrinsicID() == Intrinsic::masked_load ||
305 F->getIntrinsicID() == Intrinsic::masked_store)) {
306 unsigned OpOffset = 0;
307 if (
F->getIntrinsicID() == Intrinsic::masked_store) {
312 Access.AccessTy = CI->getArgOperand(0)->getType();
313 Access.IsWrite =
true;
317 Access.AccessTy = CI->getType();
318 Access.IsWrite =
false;
321 auto *
BasePtr = CI->getOperand(0 + OpOffset);
322 Access.MaybeMask = CI->getOperand(2 + OpOffset);
332 Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
340 if (Access.Addr->isSwiftError())
344 auto *
Addr = Access.Addr->stripInBoundsOffsets();
348 if (GV->hasSection()) {
358 if (GV->getName().startswith(
"__llvm"))
363 Access.TypeSize =
DL.getTypeStoreSizeInBits(Access.AccessTy);
369 Type *AccessTy,
bool IsWrite) {
370 auto *VTy = cast<FixedVectorType>(AccessTy);
371 uint64_t ElemTypeSize =
DL.getTypeStoreSizeInBits(VTy->getScalarType());
372 unsigned Num = VTy->getNumElements();
374 for (
unsigned Idx = 0;
Idx < Num; ++
Idx) {
375 Value *InstrumentedAddress =
nullptr;
377 if (
auto *
Vector = dyn_cast<ConstantVector>(Mask)) {
379 if (
auto *Masked = dyn_cast<ConstantInt>(
Vector->getOperand(
Idx))) {
380 if (Masked->isZero())
390 InsertBefore = ThenTerm;
394 InstrumentedAddress =
396 instrumentAddress(
I, InsertBefore, InstrumentedAddress, ElemTypeSize,
402 InterestingMemoryAccess &Access) {
406 ++NumSkippedStackWrites;
408 ++NumSkippedStackReads;
413 NumInstrumentedWrites++;
415 NumInstrumentedReads++;
417 if (Access.MaybeMask) {
419 Access.AccessTy, Access.IsWrite);
424 instrumentAddress(
I,
I, Access.Addr, Access.TypeSize, Access.IsWrite);
428void MemProfiler::instrumentAddress(
Instruction *OrigIns,
435 IRB.
CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
443 Value *ShadowPtr = memToShadow(AddrLong, IRB);
447 ShadowValue = IRB.
CreateAdd(ShadowValue, Inc);
454 dyn_cast_or_null<MDString>(M.getModuleFlag(
"MemProfProfileFilename"));
455 if (!MemProfFilename)
458 "Unexpected MemProfProfileFilename metadata with empty string");
460 M.getContext(), MemProfFilename->
getString(),
true);
462 M, ProfileNameConst->
getType(),
true,
464 Triple TT(M.getTargetTriple());
465 if (TT.supportsCOMDAT()) {
471bool ModuleMemProfiler::instrumentModule(
Module &M) {
474 std::string VersionCheckName =
477 std::tie(MemProfCtorFunction, std::ignore) =
480 {}, VersionCheckName);
482 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
490void MemProfiler::initializeCallbacks(
Module &M) {
493 for (
size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
494 const std::string TypeStr = AccessIsWrite ?
"store" :
"load";
498 MemProfMemoryAccessCallbackSized[AccessIsWrite] =
502 MemProfMemoryAccessCallback[AccessIsWrite] =
506 MemProfMemmove =
M.getOrInsertFunction(
517bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(
Function &
F) {
525 if (
F.getName().find(
" load]") != std::string::npos) {
535bool MemProfiler::insertDynamicShadowAtFunctionEntry(
Function &
F) {
537 Value *GlobalDynamicAddress =
F.getParent()->getOrInsertGlobal(
540 cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(
true);
541 DynamicShadowOffset = IRB.
CreateLoad(IntptrTy, GlobalDynamicAddress);
545bool MemProfiler::instrumentFunction(
Function &
F) {
550 if (
F.getName().startswith(
"__memprof_"))
553 bool FunctionModified =
false;
558 if (maybeInsertMemProfInitAtFunctionEntry(
F))
559 FunctionModified =
true;
563 initializeCallbacks(*
F.getParent());
569 for (
auto &Inst : BB) {
570 if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
575 if (ToInstrument.
empty()) {
576 LLVM_DEBUG(
dbgs() <<
"MEMPROF done instrumenting: " << FunctionModified
577 <<
" " <<
F <<
"\n");
579 return FunctionModified;
582 FunctionModified |= insertDynamicShadowAtFunctionEntry(
F);
584 int NumInstrumented = 0;
585 for (
auto *Inst : ToInstrument) {
588 std::optional<InterestingMemoryAccess> Access =
589 isInterestingMemoryAccess(Inst);
591 instrumentMop(Inst,
F.getParent()->getDataLayout(), *Access);
593 instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
598 if (NumInstrumented > 0)
599 FunctionModified =
true;
601 LLVM_DEBUG(
dbgs() <<
"MEMPROF done instrumenting: " << FunctionModified <<
" "
604 return FunctionModified;
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< int > ClDebugMin("asan-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("asan-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__asan_"))
static cl::opt< bool > ClInsertVersionCheck("asan-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
static cl::opt< bool > ClInstrumentWrites("asan-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
static cl::opt< int > ClDebugMax("asan-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
static cl::opt< bool > ClStack("asan-stack", cl::desc("Handle stack memory"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClInstrumentAtomics("asan-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
static cl::opt< int > ClMappingScale("asan-mapping-scale", cl::desc("scale of asan shadow mapping"), cl::Hidden, cl::init(0))
static cl::opt< std::string > ClDebugFunc("asan-debug-func", cl::Hidden, cl::desc("Debug func"))
static cl::opt< bool > ClInstrumentReads("asan-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask, Instruction *I, Value *Addr, MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite, Value *SizeArgument, bool UseCalls, uint32_t Exp)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
constexpr char MemProfVersionCheckNamePrefix[]
static cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority
static cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
constexpr char MemProfShadowMemoryDynamicAddress[]
constexpr uint64_t MemProfCtorAndDtorPriority
constexpr int LLVM_MEM_PROFILER_VERSION
static cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
static cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
constexpr char MemProfInitName[]
constexpr char MemProfFilenameVar[]
static cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
constexpr uint64_t DefaultShadowGranularity
constexpr uint64_t DefaultShadowScale
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
constexpr char MemProfModuleCtorName[]
static cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
static cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
static cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
static cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
static cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
static cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowGranularity))
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A container for analyses that lazily runs them and caches their results.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void setComdat(Comdat *C)
void setLinkage(LinkageTypes LT)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Type * getVoidTy()
Fetch the type representing void.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
StringRef getString() const
This is the common base class for memset/memcpy/memmove.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A Module instance is used to store all the information related to an LLVM module.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type * > InitArgTypes, bool Weak=false)
std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function, and calls sanitizer's init function from it.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...