MemProfiler.cpp File Reference

#include "llvm/Transforms/Instrumentation/MemProfiler.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BLAKE3.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <map>
#include <set>

Go to the source code of this file.

Classes
struct	AllocMatchInfo

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE   "memprof"

Functions
	STATISTIC (NumInstrumentedReads, "Number of instrumented reads")
	STATISTIC (NumInstrumentedWrites, "Number of instrumented writes")
	STATISTIC (NumSkippedStackReads, "Number of non-instrumented stack reads")
	STATISTIC (NumSkippedStackWrites, "Number of non-instrumented stack writes")
	STATISTIC (NumOfMemProfMissing, "Number of functions without memory profile.")
	STATISTIC (NumOfMemProfMismatch, "Number of functions having mismatched memory profile hash.")
	STATISTIC (NumOfMemProfFunc, "Number of functions having valid memory profile.")
	STATISTIC (NumOfMemProfAllocContextProfiles, "Number of alloc contexts in memory profile.")
	STATISTIC (NumOfMemProfCallSiteProfiles, "Number of callsites in memory profile.")
	STATISTIC (NumOfMemProfMatchedAllocContexts, "Number of matched memory profile alloc contexts.")
	STATISTIC (NumOfMemProfMatchedAllocs, "Number of matched memory profile allocs.")
	STATISTIC (NumOfMemProfMatchedCallSites, "Number of matched memory profile callsites.")
void	createProfileFileNameVar (Module &M)
void	createMemprofHistogramFlagVar (Module &M)
static void	addCallsiteMetadata (Instruction &I, std::vector< uint64_t > &InlinedCallStack, LLVMContext &Ctx)
static uint64_t	computeStackId (GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)
static uint64_t	computeStackId (const memprof::Frame &Frame)
static uint64_t	computeFullStackId (const std::vector< memprof::Frame > &CallStack)
static AllocationType	addCallStack (CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo)
static bool	stackFrameIncludesInlinedCallStack (ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack, unsigned StartIndex=0)
static bool	isAllocationWithHotColdVariant (Function *Callee, const TargetLibraryInfo &TLI)
static void	readMemprof (Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo)

Variables
cl::opt< bool >	llvm::PGOWarnMissing
cl::opt< bool >	llvm::NoPGOWarnMismatchComdatWeak
constexpr int	LLVM_MEM_PROFILER_VERSION = 1
constexpr uint64_t	DefaultMemGranularity = 64
constexpr uint64_t	HistogramGranularity = 8
constexpr uint64_t	DefaultShadowScale = 3
constexpr char	MemProfModuleCtorName [] = "memprof.module_ctor"
constexpr uint64_t	MemProfCtorAndDtorPriority = 1
constexpr uint64_t	MemProfEmscriptenCtorAndDtorPriority = 50
constexpr char	MemProfInitName [] = "__memprof_init"
constexpr char	MemProfVersionCheckNamePrefix []
constexpr char	MemProfShadowMemoryDynamicAddress []
constexpr char	MemProfFilenameVar [] = "__memprof_profile_filename"
constexpr char	MemProfHistogramFlagVar [] = "__memprof_histogram"
static cl::opt< bool >	ClInsertVersionCheck ("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
static cl::opt< bool >	ClInstrumentReads ("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
static cl::opt< bool >	ClInstrumentWrites ("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
static cl::opt< bool >	ClInstrumentAtomics ("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
static cl::opt< bool >	ClUseCalls ("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
static cl::opt< std::string >	ClMemoryAccessCallbackPrefix ("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
static cl::opt< int >	ClMappingScale ("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
static cl::opt< int >	ClMappingGranularity ("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity))
static cl::opt< bool >	ClStack ("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
static cl::opt< int >	ClDebug ("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
static cl::opt< std::string >	ClDebugFunc ("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
static cl::opt< int >	ClDebugMin ("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
static cl::opt< int >	ClDebugMax ("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
static cl::opt< bool >	ClMemProfMatchHotColdNew ("memprof-match-hot-cold-new", cl::desc("Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false))
static cl::opt< bool >	ClHistogram ("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false))
static cl::opt< bool >	ClPrintMemProfMatchInfo ("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false))
cl::opt< bool >	MemProfReportHintedSizes

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "memprof"

Definition at line 52 of file MemProfiler.cpp.

Function Documentation

addCallsiteMetadata()

static void addCallsiteMetadata ( Instruction &  I, std::vector< uint64_t > &  InlinedCallStack, LLVMContext &  Ctx  )

static

Definition at line 682 of file MemProfiler.cpp.

References llvm::memprof::buildCallstackMetadata(), and I.

Referenced by readMemprof().

addCallStack()

static AllocationType addCallStack ( CallStackTrie &  AllocTrie, const AllocationInfo *  AllocInfo  )

static

Definition at line 718 of file MemProfiler.cpp.

References llvm::memprof::CallStackTrie::addCallStack(), assert(), computeStackId(), llvm::memprof::getAllocType(), MemProfReportHintedSizes, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by readMemprof().

computeFullStackId()

static uint64_t computeFullStackId ( const std::vector< memprof::Frame > &  CallStack )

static

Definition at line 707 of file MemProfiler.cpp.

References llvm::HashBuilder< HasherT, Endianness >::add(), F, llvm::HashBuilderBase< HasherT >::final(), and llvm::little.

Referenced by readMemprof().

computeStackId() [1/2]

static uint64_t computeStackId ( const memprof::Frame &  Frame )

static

Definition at line 700 of file MemProfiler.cpp.

References llvm::memprof::Frame::Column, computeStackId(), llvm::memprof::Frame::Function, and llvm::memprof::Frame::LineOffset.

computeStackId() [2/2]

static uint64_t computeStackId ( GlobalValue::GUID  Function, uint32_t  LineOffset, uint32_t  Column  )

static

Definition at line 689 of file MemProfiler.cpp.

References llvm::HashBuilder< HasherT, Endianness >::add(), llvm::HashBuilderBase< HasherT >::final(), and llvm::little.

Referenced by addCallStack(), computeStackId(), readMemprof(), and stackFrameIncludesInlinedCallStack().

createMemprofHistogramFlagVar()

void createMemprofHistogramFlagVar

(

Module &

M

)

Definition at line 535 of file MemProfiler.cpp.

References llvm::appendToCompilerUsed(), ClHistogram, llvm::GlobalValue::ExternalLinkage, llvm::Type::getInt1Ty(), llvm::Constant::getIntegerValue(), MemProfHistogramFlagVar, and llvm::GlobalValue::WeakAnyLinkage.

createProfileFileNameVar()

void createProfileFileNameVar

(

Module &

M

)

Definition at line 514 of file MemProfiler.cpp.

References assert(), llvm::StringRef::empty(), llvm::GlobalValue::ExternalLinkage, llvm::MDString::getString(), llvm::ConstantDataArray::getString(), llvm::Value::getType(), MemProfFilenameVar, llvm::GlobalObject::setComdat(), llvm::GlobalValue::setLinkage(), and llvm::GlobalValue::WeakAnyLinkage.

isAllocationWithHotColdVariant()

static bool isAllocationWithHotColdVariant ( Function *  Callee, const TargetLibraryInfo &  TLI  )

static

Definition at line 757 of file MemProfiler.cpp.

References ClMemProfMatchHotColdNew, and llvm::TargetLibraryInfo::getLibFunc().

Referenced by readMemprof().

readMemprof()

static void readMemprof ( Module &  M, Function &  F, IndexedInstrProfReader *  MemProfReader, const TargetLibraryInfo &  TLI, std::map< uint64_t, AllocMatchInfo > &  FullStackIdToAllocMatchInfo  )

static

Definition at line 799 of file MemProfiler.cpp.

References addCallsiteMetadata(), addCallStack(), assert(), llvm::GlobalValue::AvailableExternallyLinkage, llvm::memprof::CallStackTrie::buildAndAttachMIBMetadata(), ClPrintMemProfMatchInfo, computeFullStackId(), computeStackId(), llvm::dbgs(), llvm::DS_Warning, llvm::memprof::CallStackTrie::empty(), F, llvm::InstrProfError::get(), llvm::GlobalValue::getGUID(), llvm::handleAllErrors(), llvm::hash_mismatch, I, Idx, isAllocationWithHotColdVariant(), LLVM_DEBUG, llvm::InstrProfError::message(), Name, llvm::NoPGOWarnMismatch, llvm::NoPGOWarnMismatchComdatWeak, llvm::PGOWarnMissing, stackFrameIncludesInlinedCallStack(), and llvm::unknown_function.

Referenced by llvm::MemProfUsePass::run().

stackFrameIncludesInlinedCallStack()

static bool stackFrameIncludesInlinedCallStack ( ArrayRef< Frame >  ProfileCallStack, ArrayRef< uint64_t >  InlinedCallStack, unsigned  StartIndex = 0  )

static

Definition at line 740 of file MemProfiler.cpp.

References llvm::ArrayRef< T >::begin(), computeStackId(), and llvm::ArrayRef< T >::end().

Referenced by readMemprof().

STATISTIC() [1/12]

STATISTIC	(	NumInstrumentedReads	,
		"Number of instrumented reads"
	)

STATISTIC() [2/12]

STATISTIC	(	NumInstrumentedWrites	,
		"Number of instrumented writes"
	)

STATISTIC() [3/12]

STATISTIC	(	NumOfMemProfAllocContextProfiles	,
		"Number of alloc contexts in memory profile."
	)

STATISTIC() [4/12]

STATISTIC	(	NumOfMemProfCallSiteProfiles	,
		"Number of callsites in memory profile."
	)

STATISTIC() [5/12]

STATISTIC	(	NumOfMemProfFunc	,
		"Number of functions having valid memory profile."
	)

STATISTIC() [6/12]

STATISTIC	(	NumOfMemProfMatchedAllocContexts	,
		"Number of matched memory profile alloc contexts."
	)

STATISTIC() [7/12]

STATISTIC	(	NumOfMemProfMatchedAllocs	,
		"Number of matched memory profile allocs."
	)

STATISTIC() [8/12]

STATISTIC	(	NumOfMemProfMatchedCallSites	,
		"Number of matched memory profile callsites."
	)

STATISTIC() [9/12]

STATISTIC	(	NumOfMemProfMismatch	,
		"Number of functions having mismatched memory profile hash."
	)

STATISTIC() [10/12]

STATISTIC	(	NumOfMemProfMissing	,
		"Number of functions without memory profile."
	)

STATISTIC() [11/12]

STATISTIC	(	NumSkippedStackReads	,
		"Number of non-instrumented stack reads"
	)

STATISTIC() [12/12]

STATISTIC	(	NumSkippedStackWrites	,
		"Number of non-instrumented stack writes"
	)

Variable Documentation

ClDebug

cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0)) ( "memprof-debug"  , cl::desc("debug")  , cl::Hidden  , cl::init(0)    )

static

ClDebugFunc

cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func")) ( "memprof-debug-func"  , cl::Hidden  , cl::desc("Debug func")    )

static

ClDebugMax

cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1)) ( "memprof-debug-max"  , cl::desc("Debug max inst")  , cl::Hidden  , cl::init(-1)    )

static

ClDebugMin

cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1)) ( "memprof-debug-min"  , cl::desc("Debug min inst")  , cl::Hidden  , cl::init(-1)    )

static

ClHistogram

cl::opt< bool > ClHistogram("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false)) ( "memprof-histogram"  , cl::desc("Collect access count histograms")  , cl::Hidden  , cl::init(false)    )

static

Referenced by createMemprofHistogramFlagVar(), and llvm::MemProfilerPass::run().

ClInsertVersionCheck

cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true)) ( "memprof-guard-against-version-mismatch"  , cl::desc("Guard against compiler/runtime version mismatch.")  , cl::Hidden  , cl::init(true)    )

static

ClInstrumentAtomics

cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true)) ( "memprof-instrument-atomics"  , cl::desc("instrument atomic instructions (rmw, cmpxchg)")  , cl::Hidden  , cl::init(true)    )

static

ClInstrumentReads

cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true)) ( "memprof-instrument-reads"  , cl::desc("instrument read instructions")  , cl::Hidden  , cl::init(true)    )

static

ClInstrumentWrites

cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true)) ( "memprof-instrument-writes"  , cl::desc("instrument write instructions")  , cl::Hidden  , cl::init(true)    )

static

ClMappingGranularity

cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity)) ( "memprof-mapping-granularity"  , cl::desc("granularity of memprof shadow mapping")  , cl::Hidden  , cl::init(DefaultMemGranularity)    )

static

Referenced by llvm::MemProfilerPass::run().

ClMappingScale

cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale)) ( "memprof-mapping-scale"  , cl::desc("scale of memprof shadow mapping")  , cl::Hidden  , cl::init(DefaultShadowScale)    )

static

ClMemoryAccessCallbackPrefix

cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_")) ( "memprof-memory-access-callback-prefix"  , cl::desc("Prefix for memory access callbacks")  , cl::Hidden  , cl::init("__memprof_")    )

static

ClMemProfMatchHotColdNew

cl::opt< bool > ClMemProfMatchHotColdNew("memprof-match-hot-cold-new", cl::desc( "Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false)) ( "memprof-match-hot-cold-new"  , cl::desc( "Match allocation profiles onto existing hot/cold operator new calls")  , cl::Hidden  , cl::init(false)    )

static

Referenced by isAllocationWithHotColdVariant().

ClPrintMemProfMatchInfo

cl::opt< bool > ClPrintMemProfMatchInfo("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false)) ( "memprof-print-match-info"  , cl::desc("Print matching stats for each allocation " "context in this module's profiles")  , cl::Hidden  , cl::init(false)    )

static

Referenced by readMemprof(), and llvm::MemProfUsePass::run().

ClStack

cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false)) ( "memprof-instrument-stack"  , cl::desc("Instrument scalar stack variables")  , cl::Hidden  , cl::init(false)    )

static

ClUseCalls

cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false)) ( "memprof-use-callbacks"  , cl::desc("Use callbacks instead of inline instrumentation sequences.")  , cl::Hidden  , cl::init(false)    )

static

DefaultMemGranularity

constexpr uint64_t DefaultMemGranularity = 64

constexpr

Definition at line 63 of file MemProfiler.cpp.

Referenced by llvm::MemProfilerPass::run().

DefaultShadowScale

constexpr uint64_t DefaultShadowScale = 3

constexpr

Definition at line 69 of file MemProfiler.cpp.

HistogramGranularity

constexpr uint64_t HistogramGranularity = 8

constexpr

Definition at line 66 of file MemProfiler.cpp.

LLVM_MEM_PROFILER_VERSION

constexpr int LLVM_MEM_PROFILER_VERSION = 1

constexpr

Definition at line 60 of file MemProfiler.cpp.

MemProfCtorAndDtorPriority

constexpr uint64_t MemProfCtorAndDtorPriority = 1

constexpr

Definition at line 72 of file MemProfiler.cpp.

MemProfEmscriptenCtorAndDtorPriority

constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50

constexpr

Definition at line 74 of file MemProfiler.cpp.

MemProfFilenameVar

constexpr char MemProfFilenameVar[] = "__memprof_profile_filename"

constexpr

Definition at line 82 of file MemProfiler.cpp.

Referenced by createProfileFileNameVar().

MemProfHistogramFlagVar

constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram"

constexpr

Definition at line 84 of file MemProfiler.cpp.

Referenced by createMemprofHistogramFlagVar().

MemProfInitName

constexpr char MemProfInitName[] = "__memprof_init"

constexpr

Definition at line 75 of file MemProfiler.cpp.

MemProfModuleCtorName

constexpr char MemProfModuleCtorName[] = "memprof.module_ctor"

constexpr

Definition at line 71 of file MemProfiler.cpp.

MemProfReportHintedSizes

cl::opt<bool> MemProfReportHintedSizes

extern

MemProfShadowMemoryDynamicAddress

constexpr char MemProfShadowMemoryDynamicAddress[]

constexpr

Initial value:

=
    "__memprof_shadow_memory_dynamic_address"

Definition at line 79 of file MemProfiler.cpp.

MemProfVersionCheckNamePrefix

constexpr char MemProfVersionCheckNamePrefix[]

constexpr

Initial value:

=
    "__memprof_version_mismatch_check_v"

Definition at line 76 of file MemProfiler.cpp.