Go to the documentation of this file.
43 #define DEBUG_TYPE "memprof"
59 "__memprof_version_mismatch_check_v";
62 "__memprof_shadow_memory_dynamic_address";
69 "memprof-guard-against-version-mismatch",
75 cl::desc(
"instrument read instructions"),
84 "memprof-instrument-atomics",
89 "memprof-use-callbacks",
90 cl::desc(
"Use callbacks instead of inline instrumentation sequences."),
95 cl::desc(
"Prefix for memory access callbacks"),
103 cl::desc(
"scale of memprof shadow mapping"),
108 cl::desc(
"granularity of memprof shadow mapping"),
112 cl::desc(
"Instrument scalar stack variables"),
129 STATISTIC(NumInstrumentedReads,
"Number of instrumented reads");
130 STATISTIC(NumInstrumentedWrites,
"Number of instrumented writes");
131 STATISTIC(NumSkippedStackReads,
"Number of non-instrumented stack reads");
132 STATISTIC(NumSkippedStackWrites,
"Number of non-instrumented stack writes");
138 struct ShadowMapping {
142 Mask = ~(Granularity - 1);
155 struct InterestingMemoryAccess {
160 Value *MaybeMask =
nullptr;
167 C = &(
M.getContext());
168 LongSize =
M.getDataLayout().getPointerSizeInBits();
179 InterestingMemoryAccess &Access);
188 bool maybeInsertMemProfInitAtFunctionEntry(
Function &
F);
189 bool insertDynamicShadowAtFunctionEntry(
Function &
F);
192 void initializeCallbacks(
Module &M);
197 ShadowMapping Mapping;
204 Value *DynamicShadowOffset =
nullptr;
215 StringRef getPassName()
const override {
return "MemProfilerFunctionPass"; }
218 MemProfiler Profiler(*
F.getParent());
219 return Profiler.instrumentFunction(
F);
223 class ModuleMemProfiler {
225 ModuleMemProfiler(
Module &M) { TargetTriple =
Triple(
M.getTargetTriple()); }
227 bool instrumentModule(
Module &);
231 ShadowMapping Mapping;
232 Function *MemProfCtorFunction =
nullptr;
235 class ModuleMemProfilerLegacyPass :
public ModulePass {
239 explicit ModuleMemProfilerLegacyPass() :
ModulePass(
ID) {
243 StringRef getPassName()
const override {
return "ModuleMemProfiler"; }
247 bool runOnModule(
Module &M)
override {
248 ModuleMemProfiler MemProfiler(M);
249 return MemProfiler.instrumentModule(M);
260 MemProfiler Profiler(
M);
261 if (Profiler.instrumentFunction(
F))
270 ModuleMemProfiler Profiler(
M);
271 if (Profiler.instrumentModule(
M))
279 "MemProfiler: profile memory allocations and accesses.",
286 return new MemProfilerLegacyPass();
292 "MemProfiler: profile memory allocations and accesses."
297 return new ModuleMemProfilerLegacyPass();
302 Shadow = IRB.
CreateAnd(Shadow, Mapping.Mask);
303 Shadow = IRB.
CreateLShr(Shadow, Mapping.Scale);
305 assert(DynamicShadowOffset);
306 return IRB.
CreateAdd(Shadow, DynamicShadowOffset);
312 if (isa<MemTransferInst>(
MI)) {
314 isa<MemMoveInst>(
MI) ? MemProfMemmove : MemProfMemcpy,
318 }
else if (isa<MemSetInst>(
MI)) {
325 MI->eraseFromParent();
329 MemProfiler::isInterestingMemoryAccess(
Instruction *
I)
const {
331 if (DynamicShadowOffset ==
I)
334 InterestingMemoryAccess Access;
336 if (
LoadInst *LI = dyn_cast<LoadInst>(
I)) {
339 Access.IsWrite =
false;
340 Access.AccessTy = LI->getType();
341 Access.Addr = LI->getPointerOperand();
345 Access.IsWrite =
true;
346 Access.AccessTy =
SI->getValueOperand()->getType();
347 Access.Addr =
SI->getPointerOperand();
351 Access.IsWrite =
true;
352 Access.AccessTy = RMW->getValOperand()->getType();
353 Access.Addr = RMW->getPointerOperand();
357 Access.IsWrite =
true;
358 Access.AccessTy = XCHG->getCompareOperand()->getType();
359 Access.Addr = XCHG->getPointerOperand();
360 }
else if (
auto *CI = dyn_cast<CallInst>(
I)) {
361 auto *
F = CI->getCalledFunction();
362 if (
F && (
F->getIntrinsicID() == Intrinsic::masked_load ||
363 F->getIntrinsicID() == Intrinsic::masked_store)) {
364 unsigned OpOffset = 0;
365 if (
F->getIntrinsicID() == Intrinsic::masked_store) {
370 Access.AccessTy = CI->getArgOperand(0)->getType();
371 Access.IsWrite =
true;
375 Access.AccessTy = CI->getType();
376 Access.IsWrite =
false;
379 auto *
BasePtr = CI->getOperand(0 + OpOffset);
380 Access.MaybeMask = CI->getOperand(2 + OpOffset);
390 Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
398 if (Access.Addr->isSwiftError())
402 auto *
Addr = Access.Addr->stripInBoundsOffsets();
406 if (GV->hasSection()) {
416 if (GV->getName().startswith(
"__llvm"))
421 Access.TypeSize =
DL.getTypeStoreSizeInBits(Access.AccessTy);
427 Type *AccessTy,
bool IsWrite) {
428 auto *VTy = cast<FixedVectorType>(AccessTy);
429 uint64_t ElemTypeSize =
DL.getTypeStoreSizeInBits(VTy->getScalarType());
430 unsigned Num = VTy->getNumElements();
432 for (
unsigned Idx = 0; Idx < Num; ++Idx) {
433 Value *InstrumentedAddress =
nullptr;
435 if (
auto *Vector = dyn_cast<ConstantVector>(
Mask)) {
437 if (
auto *Masked = dyn_cast<ConstantInt>(
Vector->getOperand(Idx))) {
438 if (Masked->isZero())
448 InsertBefore = ThenTerm;
452 InstrumentedAddress =
454 instrumentAddress(
I, InsertBefore, InstrumentedAddress, ElemTypeSize,
460 InterestingMemoryAccess &Access) {
464 ++NumSkippedStackWrites;
466 ++NumSkippedStackReads;
471 NumInstrumentedWrites++;
473 NumInstrumentedReads++;
475 if (Access.MaybeMask) {
477 Access.AccessTy, Access.IsWrite);
482 instrumentAddress(
I,
I, Access.Addr, Access.TypeSize, Access.IsWrite);
486 void MemProfiler::instrumentAddress(
Instruction *OrigIns,
493 IRB.
CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
501 Value *ShadowPtr = memToShadow(AddrLong, IRB);
505 ShadowValue = IRB.
CreateAdd(ShadowValue, Inc);
512 dyn_cast_or_null<MDString>(
M.getModuleFlag(
"MemProfProfileFilename"));
513 if (!MemProfFilename)
516 "Unexpected MemProfProfileFilename metadata with empty string");
518 M.getContext(), MemProfFilename->
getString(),
true);
520 M, ProfileNameConst->
getType(),
true,
522 Triple TT(
M.getTargetTriple());
523 if (TT.supportsCOMDAT()) {
529 bool ModuleMemProfiler::instrumentModule(
Module &M) {
532 std::string VersionCheckName =
535 std::tie(MemProfCtorFunction, std::ignore) =
538 {}, VersionCheckName);
540 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
548 void MemProfiler::initializeCallbacks(
Module &M) {
551 for (
size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
552 const std::string TypeStr = AccessIsWrite ?
"store" :
"load";
556 MemProfMemoryAccessCallbackSized[AccessIsWrite] =
560 MemProfMemoryAccessCallback[AccessIsWrite] =
564 MemProfMemmove =
M.getOrInsertFunction(
575 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(
Function &
F) {
583 if (
F.getName().find(
" load]") != std::string::npos) {
593 bool MemProfiler::insertDynamicShadowAtFunctionEntry(
Function &
F) {
595 Value *GlobalDynamicAddress =
F.getParent()->getOrInsertGlobal(
598 cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(
true);
599 DynamicShadowOffset = IRB.
CreateLoad(IntptrTy, GlobalDynamicAddress);
603 bool MemProfiler::instrumentFunction(
Function &
F) {
608 if (
F.getName().startswith(
"__memprof_"))
611 bool FunctionModified =
false;
616 if (maybeInsertMemProfInitAtFunctionEntry(
F))
617 FunctionModified =
true;
621 initializeCallbacks(*
F.getParent());
627 for (
auto &Inst :
BB) {
628 if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
629 ToInstrument.push_back(&Inst);
633 if (ToInstrument.empty()) {
634 LLVM_DEBUG(
dbgs() <<
"MEMPROF done instrumenting: " << FunctionModified
635 <<
" " <<
F <<
"\n");
637 return FunctionModified;
640 FunctionModified |= insertDynamicShadowAtFunctionEntry(
F);
642 int NumInstrumented = 0;
643 for (
auto *Inst : ToInstrument) {
647 isInterestingMemoryAccess(Inst);
649 instrumentMop(Inst,
F.getParent()->getDataLayout(), *Access);
651 instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
656 if (NumInstrumented > 0)
657 FunctionModified =
true;
659 LLVM_DEBUG(
dbgs() <<
"MEMPROF done instrumenting: " << FunctionModified <<
" "
662 return FunctionModified;
A set of analyses that are preserved following a run of a transformation pass.
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void initializeMemProfilerLegacyPassPass(PassRegistry &)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
void setComdat(Comdat *C)
This is an optimization pass for GlobalISel generic memory operations.
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
static cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
A parsed version of the target data layout string in and methods for querying it.
static cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Triple - Helper class for working with autoconf configuration names.
constexpr uint64_t DefaultShadowGranularity
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
The instances of the Type class are immutable: once they are created, they are never changed.
constexpr uint64_t DefaultShadowScale
This is the common base class for memset/memcpy/memmove.
constexpr int LLVM_MEM_PROFILER_VERSION
So we should use XX3Form_Rcr to implement intrinsic Convert DP outs ins xscvdpsp No builtin are required Round &Convert QP DP(dword[1] is set to zero) No builtin are required Round to Quad Precision because you need to assign rounding mode in instruction Provide builtin(set f128:$vT,(int_ppc_vsx_xsrqpi f128:$vB))(set f128 yields< n x< ty > >< result > yields< ty >< result > No builtin are required Load Store Vector
static cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
We currently generate a but we really shouldn eax ecx xorl edx divl ecx eax divl ecx movl eax ret A similar code sequence works for division We currently compile i32 v2 eax eax jo LBB1_2 and
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
FunctionPass * createMemProfilerFunctionPass()
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
(vector float) vec_cmpeq(*A, *B) C
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Represent the analysis usage information of a pass.
constexpr char MemProfInitName[]
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
STATISTIC(NumFunctions, "Total number of functions")
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
static cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
static cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
static cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask, Instruction *I, Value *Addr, MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite, Value *SizeArgument, bool UseCalls, uint32_t Exp)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof", "MemProfiler: profile memory allocations and accesses.", false, false) INITIALIZE_PASS_END(MemProfilerLegacyPass
constexpr char MemProfModuleCtorName[]
constexpr char MemProfVersionCheckNamePrefix[]
An instruction for storing to memory.
This is an important base class in LLVM.
constexpr LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, StringRef VersionCheckName=StringRef())
Creates sanitizer constructor function, and calls sanitizer's init function from it.
static cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
This is an important class for using LLVM in a threaded context.
FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type * > InitArgTypes)
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
initializer< Ty > init(const Ty &Val)
static cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority
void setLinkage(LinkageTypes LT)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
static cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StandardInstrumentations SI(Debug, VerifyEach)
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
A Module instance is used to store all the information related to an LLVM module.
StringRef - Represent a constant reference to a string, i.e.
Type * getType() const
All values are typed, get the type of this value.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
@ AvailableExternallyLinkage
Available for inspection, not emission.
An instruction for reading from memory.
an instruction that atomically reads a memory location, combines it with another value,...
static bool runOnFunction(Function &F, bool PostInlining)
static IntegerType * getInt64Ty(LLVMContext &C)
ModulePass * createModuleMemProfilerLegacyPassPass()
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void initializeModuleMemProfilerLegacyPassPass(PassRegistry &)
constexpr char MemProfFilenameVar[]
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module", "MemProfiler: profile memory allocations and accesses." "ModulePass", false, false) ModulePass *llvm
static cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowGranularity))
constexpr uint64_t MemProfCtorAndDtorPriority
constexpr char MemProfShadowMemoryDynamicAddress[]
static cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
@ ExternalLinkage
Externally visible function.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
static cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
std::string to_string(const T &Value)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
A container for analyses that lazily runs them and caches their results.
FunctionPass class - This class is used to implement most global optimizations.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Type * getVoidTy()
Fetch the type representing void.
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
StringRef getString() const
LLVM Value Representation.
An instruction that atomically checks whether a specified value is in a memory location,...
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)