42 #define DEBUG_TYPE "memprof" 58 "__memprof_version_mismatch_check_v";
61 "__memprof_shadow_memory_dynamic_address";
68 "memprof-guard-against-version-mismatch",
74 cl::desc(
"instrument read instructions"),
83 "memprof-instrument-atomics",
88 "memprof-use-callbacks",
89 cl::desc(
"Use callbacks instead of inline instrumentation sequences."),
94 cl::desc(
"Prefix for memory access callbacks"),
102 cl::desc(
"scale of memprof shadow mapping"),
107 cl::desc(
"granularity of memprof shadow mapping"),
124 STATISTIC(NumInstrumentedReads,
"Number of instrumented reads");
125 STATISTIC(NumInstrumentedWrites,
"Number of instrumented writes");
131 struct ShadowMapping {
135 Mask = ~(Granularity - 1);
143 static uint64_t getCtorAndDtorPriority(
Triple &TargetTriple) {
148 struct InterestingMemoryAccess {
153 Value *MaybeMask =
nullptr;
160 C = &(
M.getContext());
161 LongSize =
M.getDataLayout().getPointerSizeInBits();
172 InterestingMemoryAccess &Access);
182 bool maybeInsertMemProfInitAtFunctionEntry(
Function &
F);
183 bool insertDynamicShadowAtFunctionEntry(
Function &
F);
186 void initializeCallbacks(
Module &M);
191 ShadowMapping Mapping;
198 Value *DynamicShadowOffset =
nullptr;
209 StringRef getPassName()
const override {
return "MemProfilerFunctionPass"; }
212 MemProfiler Profiler(*
F.getParent());
213 return Profiler.instrumentFunction(
F);
217 class ModuleMemProfiler {
219 ModuleMemProfiler(
Module &M) { TargetTriple =
Triple(
M.getTargetTriple()); }
221 bool instrumentModule(
Module &);
225 ShadowMapping Mapping;
226 Function *MemProfCtorFunction =
nullptr;
229 class ModuleMemProfilerLegacyPass :
public ModulePass {
233 explicit ModuleMemProfilerLegacyPass() :
ModulePass(
ID) {
237 StringRef getPassName()
const override {
return "ModuleMemProfiler"; }
241 bool runOnModule(
Module &M)
override {
242 ModuleMemProfiler MemProfiler(M);
243 return MemProfiler.instrumentModule(M);
254 MemProfiler Profiler(M);
255 if (Profiler.instrumentFunction(
F))
266 ModuleMemProfiler Profiler(M);
267 if (Profiler.instrumentModule(M))
275 "MemProfiler: profile memory allocations and accesses.",
278 "MemProfiler:
profile memory allocations and accesses.",
282 return new MemProfilerLegacyPass();
288 "MemProfiler: profile memory allocations and accesses." 293 return new ModuleMemProfilerLegacyPass();
298 Shadow = IRB.
CreateAnd(Shadow, Mapping.Mask);
299 Shadow = IRB.
CreateLShr(Shadow, Mapping.Scale);
301 assert(DynamicShadowOffset);
302 return IRB.
CreateAdd(Shadow, DynamicShadowOffset);
308 if (isa<MemTransferInst>(
MI)) {
310 isa<MemMoveInst>(
MI) ? MemProfMemmove : MemProfMemcpy,
314 }
else if (isa<MemSetInst>(
MI)) {
321 MI->eraseFromParent();
325 MemProfiler::isInterestingMemoryAccess(
Instruction *
I)
const {
327 if (DynamicShadowOffset ==
I)
330 InterestingMemoryAccess Access;
333 if (
LoadInst *LI = dyn_cast<LoadInst>(
I)) {
336 Access.IsWrite =
false;
337 Access.TypeSize =
DL.getTypeStoreSizeInBits(LI->getType());
338 Access.Alignment = LI->getAlignment();
339 Access.Addr = LI->getPointerOperand();
343 Access.IsWrite =
true;
345 DL.getTypeStoreSizeInBits(
SI->getValueOperand()->getType());
346 Access.Alignment =
SI->getAlignment();
347 Access.Addr =
SI->getPointerOperand();
351 Access.IsWrite =
true;
353 DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
354 Access.Alignment = 0;
355 Access.Addr = RMW->getPointerOperand();
359 Access.IsWrite =
true;
361 DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
362 Access.Alignment = 0;
363 Access.Addr = XCHG->getPointerOperand();
364 }
else if (
auto *CI = dyn_cast<CallInst>(
I)) {
365 auto *
F = CI->getCalledFunction();
366 if (
F && (
F->getIntrinsicID() == Intrinsic::masked_load ||
367 F->getIntrinsicID() == Intrinsic::masked_store)) {
368 unsigned OpOffset = 0;
369 if (
F->getIntrinsicID() == Intrinsic::masked_store) {
374 Access.IsWrite =
true;
378 Access.IsWrite =
false;
381 auto *
BasePtr = CI->getOperand(0 + OpOffset);
382 auto *Ty = cast<PointerType>(
BasePtr->getType())->getElementType();
383 Access.TypeSize =
DL.getTypeStoreSizeInBits(Ty);
384 if (
auto *AlignmentConstant =
385 dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
386 Access.Alignment = (
unsigned)AlignmentConstant->getZExtValue();
388 Access.Alignment = 1;
389 Access.MaybeMask = CI->getOperand(2 + OpOffset);
399 Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
407 if (Access.Addr->isSwiftError())
417 auto *VTy = cast<FixedVectorType>(
418 cast<PointerType>(
Addr->getType())->getElementType());
419 uint64_t ElemTypeSize =
DL.getTypeStoreSizeInBits(VTy->getScalarType());
420 unsigned Num = VTy->getNumElements();
422 for (
unsigned Idx = 0; Idx < Num; ++Idx) {
423 Value *InstrumentedAddress =
nullptr;
425 if (
auto *Vector = dyn_cast<ConstantVector>(
Mask)) {
427 if (
auto *Masked = dyn_cast<ConstantInt>(
Vector->getOperand(Idx))) {
428 if (Masked->isZero())
438 InsertBefore = ThenTerm;
442 InstrumentedAddress =
444 instrumentAddress(
I, InsertBefore, InstrumentedAddress, ElemTypeSize,
450 InterestingMemoryAccess &Access) {
452 NumInstrumentedWrites++;
454 NumInstrumentedReads++;
456 if (Access.MaybeMask) {
458 Access.Alignment, Access.TypeSize,
464 instrumentAddress(
I,
I, Access.Addr, Access.TypeSize, Access.IsWrite);
468 void MemProfiler::instrumentAddress(
Instruction *OrigIns,
475 IRB.
CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
483 Value *ShadowPtr = memToShadow(AddrLong, IRB);
487 ShadowValue = IRB.
CreateAdd(ShadowValue, Inc);
494 dyn_cast_or_null<MDString>(M.getModuleFlag(
"MemProfProfileFilename"));
495 if (!MemProfFilename)
498 "Unexpected MemProfProfileFilename metadata with empty string");
500 M.getContext(), MemProfFilename->
getString(),
true);
502 M, ProfileNameConst->getType(),
true,
504 Triple TT(M.getTargetTriple());
505 if (TT.supportsCOMDAT()) {
511 bool ModuleMemProfiler::instrumentModule(
Module &M) {
514 std::string VersionCheckName =
517 std::tie(MemProfCtorFunction, std::ignore) =
520 {}, VersionCheckName);
522 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
530 void MemProfiler::initializeCallbacks(
Module &M) {
533 for (
size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
534 const std::string TypeStr = AccessIsWrite ?
"store" :
"load";
538 MemProfMemoryAccessCallbackSized[AccessIsWrite] =
542 MemProfMemoryAccessCallback[AccessIsWrite] =
546 MemProfMemmove =
M.getOrInsertFunction(
557 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(
Function &
F) {
565 if (
F.getName().find(
" load]") != std::string::npos) {
575 bool MemProfiler::insertDynamicShadowAtFunctionEntry(
Function &
F) {
577 Value *GlobalDynamicAddress =
F.getParent()->getOrInsertGlobal(
580 cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(
true);
581 DynamicShadowOffset = IRB.
CreateLoad(IntptrTy, GlobalDynamicAddress);
585 bool MemProfiler::instrumentFunction(
Function &
F) {
590 if (
F.getName().startswith(
"__memprof_"))
593 bool FunctionModified =
false;
598 if (maybeInsertMemProfInitAtFunctionEntry(
F))
599 FunctionModified =
true;
603 initializeCallbacks(*
F.getParent());
605 FunctionModified |= insertDynamicShadowAtFunctionEntry(
F);
611 for (
auto &Inst : BB) {
612 if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
617 int NumInstrumented = 0;
618 for (
auto *Inst : ToInstrument) {
622 isInterestingMemoryAccess(Inst);
624 instrumentMop(Inst,
F.getParent()->getDataLayout(), *Access);
626 instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
631 if (NumInstrumented > 0)
632 FunctionModified =
true;
634 LLVM_DEBUG(
dbgs() <<
"MEMPROF done instrumenting: " << FunctionModified <<
" " 637 return FunctionModified;
static cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
constexpr uint64_t DefaultShadowGranularity
static cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
A parsed version of the target data layout string in and methods for querying it.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
constexpr int LLVM_MEM_PROFILER_VERSION
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
This class represents lattice values for constants.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, StringRef VersionCheckName=StringRef())
Creates sanitizer constructor function, and calls sanitizer's init function from it.
A Module instance is used to store all the information related to an LLVM module.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Available for inspection, not emission.
void push_back(const T &Elt)
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask, Instruction *I, Value *Addr, MaybeAlign Alignment, unsigned Granularity, uint32_t TypeSize, bool IsWrite, Value *SizeArgument, bool UseCalls, uint32_t Exp)
Externally visible function.
STATISTIC(NumFunctions, "Total number of functions")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
An instruction for reading from memory.
static IntegerType * getInt64Ty(LLVMContext &C)
an instruction that atomically reads a memory location, combines it with another value,...
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type * > InitArgTypes)
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Type * getVoidTy()
Fetch the type representing void.
INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof", "MemProfiler: profile memory allocations and accesses.", false, false) INITIALIZE_PASS_END(MemProfilerLegacyPass
constexpr uint64_t DefaultShadowScale
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
An instruction for storing to memory.
static cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
constexpr char MemProfInitName[]
StringRef getString() const
constexpr char MemProfModuleCtorName[]
static bool runOnFunction(Function &F, bool PostInlining)
initializer< Ty > init(const Ty &Val)
A set of analyses that are preserved following a run of a transformation pass.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
The instances of the Type class are immutable: once they are created, they are never changed.
static cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
This is an important class for using LLVM in a threaded context.
static cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
This is an important base class in LLVM.
FunctionPass * createMemProfilerFunctionPass()
static cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
Represent the analysis usage information of a pass.
static cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
FunctionPass class - This class is used to implement most global optimizations.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
constexpr char MemProfVersionCheckNamePrefix[]
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Triple - Helper class for working with autoconf configuration names.
This is the common base class for memset/memcpy/memmove.
void initializeModuleMemProfilerLegacyPassPass(PassRegistry &)
static cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Module.h This file contains the declarations for the Module class.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority
constexpr char MemProfFilenameVar[]
static cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
pgo instr Read PGO instrumentation profile
void initializeMemProfilerLegacyPassPass(PassRegistry &)
Value * CreateGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Keep one copy of named function when linking (weak)
static cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowGranularity))
const std::string to_string(const T &Value)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
StringRef - Represent a constant reference to a string, i.e.
A container for analyses that lazily runs them and caches their results.
INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module", "MemProfiler: profile memory allocations and accesses." "ModulePass", false, false) ModulePass *llvm
constexpr char MemProfShadowMemoryDynamicAddress[]
constexpr uint64_t MemProfCtorAndDtorPriority
ModulePass * createModuleMemProfilerLegacyPassPass()
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL