LLVM 20.0.0git
Classes | Namespaces | Macros | Functions | Variables
MemProfiler.cpp File Reference
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BLAKE3.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LongestCommonSequence.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <map>
#include <set>

Go to the source code of this file.

Classes

struct  AllocMatchInfo
 

Namespaces

namespace  llvm
 This is an optimization pass for GlobalISel generic memory operations.
 

Macros

#define DEBUG_TYPE   "memprof"
 

Functions

 STATISTIC (NumInstrumentedReads, "Number of instrumented reads")
 
 STATISTIC (NumInstrumentedWrites, "Number of instrumented writes")
 
 STATISTIC (NumSkippedStackReads, "Number of non-instrumented stack reads")
 
 STATISTIC (NumSkippedStackWrites, "Number of non-instrumented stack writes")
 
 STATISTIC (NumOfMemProfMissing, "Number of functions without memory profile.")
 
 STATISTIC (NumOfMemProfMismatch, "Number of functions having mismatched memory profile hash.")
 
 STATISTIC (NumOfMemProfFunc, "Number of functions having valid memory profile.")
 
 STATISTIC (NumOfMemProfAllocContextProfiles, "Number of alloc contexts in memory profile.")
 
 STATISTIC (NumOfMemProfCallSiteProfiles, "Number of callsites in memory profile.")
 
 STATISTIC (NumOfMemProfMatchedAllocContexts, "Number of matched memory profile alloc contexts.")
 
 STATISTIC (NumOfMemProfMatchedAllocs, "Number of matched memory profile allocs.")
 
 STATISTIC (NumOfMemProfMatchedCallSites, "Number of matched memory profile callsites.")
 
void createProfileFileNameVar (Module &M)
 
void createMemprofHistogramFlagVar (Module &M)
 
void createMemprofDefaultOptionsVar (Module &M)
 
static void addCallsiteMetadata (Instruction &I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx)
 
static uint64_t computeStackId (GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)
 
static uint64_t computeStackId (const memprof::Frame &Frame)
 
static uint64_t computeFullStackId (ArrayRef< Frame > CallStack)
 
static AllocationType addCallStack (CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo, uint64_t FullStackId)
 
static bool stackFrameIncludesInlinedCallStack (ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack)
 
static bool isAllocationWithHotColdVariant (const Function *Callee, const TargetLibraryInfo &TLI)
 
static void undriftMemProfRecord (const DenseMap< uint64_t, LocToLocMap > &UndriftMaps, memprof::MemProfRecord &MemProfRec)
 
static void readMemprof (Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo, DenseMap< uint64_t, LocToLocMap > &UndriftMaps)
 

Variables

cl::opt< boolllvm::PGOWarnMissing
 
cl::opt< boolllvm::NoPGOWarnMismatchComdatWeak
 
constexpr int LLVM_MEM_PROFILER_VERSION = 1
 
constexpr uint64_t DefaultMemGranularity = 64
 
constexpr uint64_t HistogramGranularity = 8
 
constexpr uint64_t DefaultShadowScale = 3
 
constexpr char MemProfModuleCtorName [] = "memprof.module_ctor"
 
constexpr uint64_t MemProfCtorAndDtorPriority = 1
 
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50
 
constexpr char MemProfInitName [] = "__memprof_init"
 
constexpr char MemProfVersionCheckNamePrefix []
 
constexpr char MemProfShadowMemoryDynamicAddress []
 
constexpr char MemProfFilenameVar [] = "__memprof_profile_filename"
 
constexpr char MemProfHistogramFlagVar [] = "__memprof_histogram"
 
static cl::opt< boolClInsertVersionCheck ("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
 
static cl::opt< boolClInstrumentReads ("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
 
static cl::opt< boolClInstrumentWrites ("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
 
static cl::opt< boolClInstrumentAtomics ("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
 
static cl::opt< boolClUseCalls ("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
 
static cl::opt< std::string > ClMemoryAccessCallbackPrefix ("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
 
static cl::opt< int > ClMappingScale ("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
 
static cl::opt< int > ClMappingGranularity ("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity))
 
static cl::opt< boolClStack ("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
 
static cl::opt< int > ClDebug ("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
 
static cl::opt< std::string > ClDebugFunc ("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
 
static cl::opt< int > ClDebugMin ("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
 
static cl::opt< int > ClDebugMax ("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
 
static cl::opt< boolClMemProfMatchHotColdNew ("memprof-match-hot-cold-new", cl::desc("Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false))
 
static cl::opt< boolClHistogram ("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false))
 
static cl::opt< boolClPrintMemProfMatchInfo ("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false))
 
static cl::opt< std::string > MemprofRuntimeDefaultOptions ("memprof-runtime-default-options", cl::desc("The default memprof options"), cl::Hidden, cl::init(""))
 
static cl::opt< boolSalvageStaleProfile ("memprof-salvage-stale-profile", cl::desc("Salvage stale MemProf profile"), cl::init(false), cl::Hidden)
 
cl::opt< boolMemProfReportHintedSizes
 
static cl::opt< unsignedMinMatchedColdBytePercent ("memprof-matching-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes matched to hint allocation cold"))
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "memprof"

Definition at line 53 of file MemProfiler.cpp.

Function Documentation

◆ addCallsiteMetadata()

static void addCallsiteMetadata ( Instruction I,
ArrayRef< uint64_t InlinedCallStack,
LLVMContext Ctx 
)
static

Definition at line 713 of file MemProfiler.cpp.

References llvm::memprof::buildCallstackMetadata(), and I.

Referenced by readMemprof().

◆ addCallStack()

static AllocationType addCallStack ( CallStackTrie AllocTrie,
const AllocationInfo AllocInfo,
uint64_t  FullStackId 
)
static

◆ computeFullStackId()

static uint64_t computeFullStackId ( ArrayRef< Frame CallStack)
static

◆ computeStackId() [1/2]

static uint64_t computeStackId ( const memprof::Frame Frame)
static

◆ computeStackId() [2/2]

static uint64_t computeStackId ( GlobalValue::GUID  Function,
uint32_t  LineOffset,
uint32_t  Column 
)
static

◆ createMemprofDefaultOptionsVar()

void createMemprofDefaultOptionsVar ( Module M)

◆ createMemprofHistogramFlagVar()

void createMemprofHistogramFlagVar ( Module M)

◆ createProfileFileNameVar()

void createProfileFileNameVar ( Module M)

◆ isAllocationWithHotColdVariant()

static bool isAllocationWithHotColdVariant ( const Function Callee,
const TargetLibraryInfo TLI 
)
static

◆ readMemprof()

static void readMemprof ( Module M,
Function F,
IndexedInstrProfReader MemProfReader,
const TargetLibraryInfo TLI,
std::map< uint64_t, AllocMatchInfo > &  FullStackIdToAllocMatchInfo,
DenseMap< uint64_t, LocToLocMap > &  UndriftMaps 
)
static

◆ stackFrameIncludesInlinedCallStack()

static bool stackFrameIncludesInlinedCallStack ( ArrayRef< Frame ProfileCallStack,
ArrayRef< uint64_t InlinedCallStack 
)
static

Definition at line 773 of file MemProfiler.cpp.

References llvm::ArrayRef< T >::begin(), computeStackId(), and llvm::ArrayRef< T >::end().

Referenced by readMemprof().

◆ STATISTIC() [1/12]

STATISTIC ( NumInstrumentedReads  ,
"Number of instrumented reads"   
)

◆ STATISTIC() [2/12]

STATISTIC ( NumInstrumentedWrites  ,
"Number of instrumented writes"   
)

◆ STATISTIC() [3/12]

STATISTIC ( NumOfMemProfAllocContextProfiles  ,
"Number of alloc contexts in memory profile."   
)

◆ STATISTIC() [4/12]

STATISTIC ( NumOfMemProfCallSiteProfiles  ,
"Number of callsites in memory profile."   
)

◆ STATISTIC() [5/12]

STATISTIC ( NumOfMemProfFunc  ,
"Number of functions having valid memory profile."   
)

◆ STATISTIC() [6/12]

STATISTIC ( NumOfMemProfMatchedAllocContexts  ,
"Number of matched memory profile alloc contexts."   
)

◆ STATISTIC() [7/12]

STATISTIC ( NumOfMemProfMatchedAllocs  ,
"Number of matched memory profile allocs."   
)

◆ STATISTIC() [8/12]

STATISTIC ( NumOfMemProfMatchedCallSites  ,
"Number of matched memory profile callsites."   
)

◆ STATISTIC() [9/12]

STATISTIC ( NumOfMemProfMismatch  ,
"Number of functions having mismatched memory profile hash."   
)

◆ STATISTIC() [10/12]

STATISTIC ( NumOfMemProfMissing  ,
"Number of functions without memory profile."   
)

◆ STATISTIC() [11/12]

STATISTIC ( NumSkippedStackReads  ,
"Number of non-instrumented stack reads"   
)

◆ STATISTIC() [12/12]

STATISTIC ( NumSkippedStackWrites  ,
"Number of non-instrumented stack writes"   
)

◆ undriftMemProfRecord()

static void undriftMemProfRecord ( const DenseMap< uint64_t, LocToLocMap > &  UndriftMaps,
memprof::MemProfRecord MemProfRec 
)
static

Variable Documentation

◆ ClDebug

cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0)) ( "memprof-debug"  ,
cl::desc("debug")  ,
cl::Hidden  ,
cl::init(0)   
)
static

◆ ClDebugFunc

cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func")) ( "memprof-debug-func"  ,
cl::Hidden  ,
cl::desc("Debug func")   
)
static

◆ ClDebugMax

cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1)) ( "memprof-debug-max"  ,
cl::desc("Debug max inst")  ,
cl::Hidden  ,
cl::init(-1)   
)
static

◆ ClDebugMin

cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1)) ( "memprof-debug-min"  ,
cl::desc("Debug min inst")  ,
cl::Hidden  ,
cl::init(-1)   
)
static

◆ ClHistogram

cl::opt< bool > ClHistogram("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false)) ( "memprof-histogram"  ,
cl::desc("Collect access count histograms")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClInsertVersionCheck

cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true)) ( "memprof-guard-against-version-mismatch"  ,
cl::desc("Guard against compiler/runtime version mismatch.")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClInstrumentAtomics

cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true)) ( "memprof-instrument-atomics"  ,
cl::desc("instrument atomic instructions (rmw, cmpxchg)")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClInstrumentReads

cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true)) ( "memprof-instrument-reads"  ,
cl::desc("instrument read instructions")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClInstrumentWrites

cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true)) ( "memprof-instrument-writes"  ,
cl::desc("instrument write instructions")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClMappingGranularity

cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity)) ( "memprof-mapping-granularity"  ,
cl::desc("granularity of memprof shadow mapping")  ,
cl::Hidden  ,
cl::init(DefaultMemGranularity  
)
static

◆ ClMappingScale

cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale)) ( "memprof-mapping-scale"  ,
cl::desc("scale of memprof shadow mapping")  ,
cl::Hidden  ,
cl::init(DefaultShadowScale  
)
static

◆ ClMemoryAccessCallbackPrefix

cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_")) ( "memprof-memory-access-callback-prefix"  ,
cl::desc("Prefix for memory access callbacks")  ,
cl::Hidden  ,
cl::init("__memprof_")   
)
static

◆ ClMemProfMatchHotColdNew

cl::opt< bool > ClMemProfMatchHotColdNew("memprof-match-hot-cold-new", cl::desc( "Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false)) ( "memprof-match-hot-cold-new"  ,
cl::desc( "Match allocation profiles onto existing hot/cold operator new calls")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClPrintMemProfMatchInfo

cl::opt< bool > ClPrintMemProfMatchInfo("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false)) ( "memprof-print-match-info"  ,
cl::desc("Print matching stats for each allocation " "context in this module's profiles")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClStack

cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false)) ( "memprof-instrument-stack"  ,
cl::desc("Instrument scalar stack variables")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClUseCalls

cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false)) ( "memprof-use-callbacks"  ,
cl::desc("Use callbacks instead of inline instrumentation sequences.")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ DefaultMemGranularity

constexpr uint64_t DefaultMemGranularity = 64
constexpr

Definition at line 64 of file MemProfiler.cpp.

Referenced by llvm::MemProfilerPass::run().

◆ DefaultShadowScale

constexpr uint64_t DefaultShadowScale = 3
constexpr

Definition at line 70 of file MemProfiler.cpp.

◆ HistogramGranularity

constexpr uint64_t HistogramGranularity = 8
constexpr

Definition at line 67 of file MemProfiler.cpp.

◆ LLVM_MEM_PROFILER_VERSION

constexpr int LLVM_MEM_PROFILER_VERSION = 1
constexpr

Definition at line 61 of file MemProfiler.cpp.

◆ MemProfCtorAndDtorPriority

constexpr uint64_t MemProfCtorAndDtorPriority = 1
constexpr

Definition at line 73 of file MemProfiler.cpp.

◆ MemProfEmscriptenCtorAndDtorPriority

constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50
constexpr

Definition at line 75 of file MemProfiler.cpp.

◆ MemProfFilenameVar

constexpr char MemProfFilenameVar[] = "__memprof_profile_filename"
constexpr

Definition at line 83 of file MemProfiler.cpp.

Referenced by createProfileFileNameVar().

◆ MemProfHistogramFlagVar

constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram"
constexpr

Definition at line 85 of file MemProfiler.cpp.

Referenced by createMemprofHistogramFlagVar().

◆ MemProfInitName

constexpr char MemProfInitName[] = "__memprof_init"
constexpr

Definition at line 76 of file MemProfiler.cpp.

◆ MemProfModuleCtorName

constexpr char MemProfModuleCtorName[] = "memprof.module_ctor"
constexpr

Definition at line 72 of file MemProfiler.cpp.

◆ MemProfReportHintedSizes

cl::opt<bool> MemProfReportHintedSizes
extern

◆ MemprofRuntimeDefaultOptions

cl::opt< std::string > MemprofRuntimeDefaultOptions("memprof-runtime-default-options", cl::desc("The default memprof options"), cl::Hidden, cl::init("")) ( "memprof-runtime-default-options"  ,
cl::desc("The default memprof options")  ,
cl::Hidden  ,
cl::init("")   
)
static

◆ MemProfShadowMemoryDynamicAddress

constexpr char MemProfShadowMemoryDynamicAddress[]
constexpr
Initial value:
=
"__memprof_shadow_memory_dynamic_address"

Definition at line 80 of file MemProfiler.cpp.

◆ MemProfVersionCheckNamePrefix

constexpr char MemProfVersionCheckNamePrefix[]
constexpr
Initial value:
=
"__memprof_version_mismatch_check_v"

Definition at line 77 of file MemProfiler.cpp.

◆ MinMatchedColdBytePercent

cl::opt< unsigned > MinMatchedColdBytePercent("memprof-matching-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes matched to hint allocation cold")) ( "memprof-matching-cold-threshold"  ,
cl::init(100)  ,
cl::Hidden  ,
cl::desc("Min percent of cold bytes matched to hint allocation cold")   
)
static

Referenced by readMemprof().

◆ SalvageStaleProfile

cl::opt< bool > SalvageStaleProfile("memprof-salvage-stale-profile", cl::desc("Salvage stale MemProf profile"), cl::init(false), cl::Hidden) ( "memprof-salvage-stale-profile"  ,
cl::desc("Salvage stale MemProf profile")  ,
cl::init(false)  ,
cl::Hidden   
)
static