LLVM 20.0.0git
|
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BLAKE3.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LongestCommonSequence.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <map>
#include <set>
Go to the source code of this file.
Classes | |
struct | AllocMatchInfo |
Namespaces | |
namespace | llvm |
This is an optimization pass for GlobalISel generic memory operations. | |
Macros | |
#define | DEBUG_TYPE "memprof" |
Functions | |
STATISTIC (NumInstrumentedReads, "Number of instrumented reads") | |
STATISTIC (NumInstrumentedWrites, "Number of instrumented writes") | |
STATISTIC (NumSkippedStackReads, "Number of non-instrumented stack reads") | |
STATISTIC (NumSkippedStackWrites, "Number of non-instrumented stack writes") | |
STATISTIC (NumOfMemProfMissing, "Number of functions without memory profile.") | |
STATISTIC (NumOfMemProfMismatch, "Number of functions having mismatched memory profile hash.") | |
STATISTIC (NumOfMemProfFunc, "Number of functions having valid memory profile.") | |
STATISTIC (NumOfMemProfAllocContextProfiles, "Number of alloc contexts in memory profile.") | |
STATISTIC (NumOfMemProfCallSiteProfiles, "Number of callsites in memory profile.") | |
STATISTIC (NumOfMemProfMatchedAllocContexts, "Number of matched memory profile alloc contexts.") | |
STATISTIC (NumOfMemProfMatchedAllocs, "Number of matched memory profile allocs.") | |
STATISTIC (NumOfMemProfMatchedCallSites, "Number of matched memory profile callsites.") | |
void | createProfileFileNameVar (Module &M) |
void | createMemprofHistogramFlagVar (Module &M) |
void | createMemprofDefaultOptionsVar (Module &M) |
static void | addCallsiteMetadata (Instruction &I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx) |
static uint64_t | computeStackId (GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column) |
static uint64_t | computeStackId (const memprof::Frame &Frame) |
static uint64_t | computeFullStackId (ArrayRef< Frame > CallStack) |
static AllocationType | addCallStack (CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo, uint64_t FullStackId) |
static bool | stackFrameIncludesInlinedCallStack (ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack) |
static bool | isAllocationWithHotColdVariant (const Function *Callee, const TargetLibraryInfo &TLI) |
static void | undriftMemProfRecord (const DenseMap< uint64_t, LocToLocMap > &UndriftMaps, memprof::MemProfRecord &MemProfRec) |
static void | readMemprof (Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo, DenseMap< uint64_t, LocToLocMap > &UndriftMaps) |
Variables | |
cl::opt< bool > | llvm::PGOWarnMissing |
cl::opt< bool > | llvm::NoPGOWarnMismatchComdatWeak |
constexpr int | LLVM_MEM_PROFILER_VERSION = 1 |
constexpr uint64_t | DefaultMemGranularity = 64 |
constexpr uint64_t | HistogramGranularity = 8 |
constexpr uint64_t | DefaultShadowScale = 3 |
constexpr char | MemProfModuleCtorName [] = "memprof.module_ctor" |
constexpr uint64_t | MemProfCtorAndDtorPriority = 1 |
constexpr uint64_t | MemProfEmscriptenCtorAndDtorPriority = 50 |
constexpr char | MemProfInitName [] = "__memprof_init" |
constexpr char | MemProfVersionCheckNamePrefix [] |
constexpr char | MemProfShadowMemoryDynamicAddress [] |
constexpr char | MemProfFilenameVar [] = "__memprof_profile_filename" |
constexpr char | MemProfHistogramFlagVar [] = "__memprof_histogram" |
static cl::opt< bool > | ClInsertVersionCheck ("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true)) |
static cl::opt< bool > | ClInstrumentReads ("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true)) |
static cl::opt< bool > | ClInstrumentWrites ("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true)) |
static cl::opt< bool > | ClInstrumentAtomics ("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true)) |
static cl::opt< bool > | ClUseCalls ("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false)) |
static cl::opt< std::string > | ClMemoryAccessCallbackPrefix ("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_")) |
static cl::opt< int > | ClMappingScale ("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale)) |
static cl::opt< int > | ClMappingGranularity ("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity)) |
static cl::opt< bool > | ClStack ("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false)) |
static cl::opt< int > | ClDebug ("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0)) |
static cl::opt< std::string > | ClDebugFunc ("memprof-debug-func", cl::Hidden, cl::desc("Debug func")) |
static cl::opt< int > | ClDebugMin ("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1)) |
static cl::opt< int > | ClDebugMax ("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1)) |
static cl::opt< bool > | ClMemProfMatchHotColdNew ("memprof-match-hot-cold-new", cl::desc("Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false)) |
static cl::opt< bool > | ClHistogram ("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false)) |
static cl::opt< bool > | ClPrintMemProfMatchInfo ("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false)) |
static cl::opt< std::string > | MemprofRuntimeDefaultOptions ("memprof-runtime-default-options", cl::desc("The default memprof options"), cl::Hidden, cl::init("")) |
static cl::opt< bool > | SalvageStaleProfile ("memprof-salvage-stale-profile", cl::desc("Salvage stale MemProf profile"), cl::init(false), cl::Hidden) |
cl::opt< bool > | MemProfReportHintedSizes |
static cl::opt< unsigned > | MinMatchedColdBytePercent ("memprof-matching-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes matched to hint allocation cold")) |
#define DEBUG_TYPE "memprof" |
Definition at line 53 of file MemProfiler.cpp.
|
static |
Definition at line 713 of file MemProfiler.cpp.
References llvm::memprof::buildCallstackMetadata(), and I.
Referenced by readMemprof().
|
static |
Definition at line 748 of file MemProfiler.cpp.
References llvm::memprof::CallStackTrie::addCallStack(), assert(), computeStackId(), llvm::memprof::getAllocType(), MemProfReportHintedSizes, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by readMemprof().
Definition at line 737 of file MemProfiler.cpp.
References llvm::HashBuilder< HasherT, Endianness >::add(), F, llvm::HashBuilderBase< HasherT >::final(), and llvm::little.
Referenced by readMemprof().
|
static |
Definition at line 731 of file MemProfiler.cpp.
References llvm::memprof::Frame::Column, computeStackId(), llvm::memprof::Frame::Function, and llvm::memprof::Frame::LineOffset.
|
static |
Definition at line 720 of file MemProfiler.cpp.
References llvm::HashBuilder< HasherT, Endianness >::add(), llvm::HashBuilderBase< HasherT >::final(), and llvm::little.
Referenced by addCallStack(), computeStackId(), readMemprof(), and stackFrameIncludesInlinedCallStack().
void createMemprofDefaultOptionsVar | ( | Module & | M | ) |
Definition at line 564 of file MemProfiler.cpp.
References llvm::GlobalValue::ExternalLinkage, llvm::Value::getName(), llvm::ConstantDataArray::getString(), llvm::Value::getType(), MemprofRuntimeDefaultOptions, llvm::GlobalObject::setComdat(), llvm::GlobalValue::setLinkage(), and llvm::GlobalValue::WeakAnyLinkage.
void createMemprofHistogramFlagVar | ( | Module & | M | ) |
Definition at line 550 of file MemProfiler.cpp.
References llvm::appendToCompilerUsed(), ClHistogram, llvm::GlobalValue::ExternalLinkage, llvm::Type::getInt1Ty(), llvm::Constant::getIntegerValue(), MemProfHistogramFlagVar, and llvm::GlobalValue::WeakAnyLinkage.
void createProfileFileNameVar | ( | Module & | M | ) |
Definition at line 529 of file MemProfiler.cpp.
References assert(), llvm::StringRef::empty(), llvm::GlobalValue::ExternalLinkage, llvm::MDString::getString(), llvm::ConstantDataArray::getString(), llvm::Value::getType(), MemProfFilenameVar, llvm::GlobalObject::setComdat(), llvm::GlobalValue::setLinkage(), and llvm::GlobalValue::WeakAnyLinkage.
|
static |
Definition at line 789 of file MemProfiler.cpp.
References ClMemProfMatchHotColdNew, and llvm::TargetLibraryInfo::getLibFunc().
Referenced by llvm::memprof::extractCallsFromIR(), and readMemprof().
|
static |
Definition at line 947 of file MemProfiler.cpp.
References addCallsiteMetadata(), addCallStack(), llvm::memprof::CallStackTrie::addSingleAllocTypeAttribute(), assert(), llvm::GlobalValue::AvailableExternallyLinkage, llvm::memprof::CallStackTrie::buildAndAttachMIBMetadata(), ClPrintMemProfMatchInfo, llvm::Cold, computeFullStackId(), computeStackId(), llvm::dbgs(), llvm::DS_Warning, llvm::memprof::CallStackTrie::empty(), F, llvm::InstrProfError::get(), llvm::GlobalValue::getGUID(), llvm::handleAllErrors(), llvm::hash_mismatch, I, Idx, isAllocationWithHotColdVariant(), LLVM_DEBUG, MemProfReportHintedSizes, llvm::InstrProfError::message(), MinMatchedColdBytePercent, Name, llvm::NoPGOWarnMismatch, llvm::NoPGOWarnMismatchComdatWeak, llvm::PGOWarnMissing, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SalvageStaleProfile, stackFrameIncludesInlinedCallStack(), undriftMemProfRecord(), and llvm::unknown_function.
Referenced by llvm::MemProfUsePass::run().
|
static |
Definition at line 773 of file MemProfiler.cpp.
References llvm::ArrayRef< T >::begin(), computeStackId(), and llvm::ArrayRef< T >::end().
Referenced by readMemprof().
STATISTIC | ( | NumInstrumentedReads | , |
"Number of instrumented reads" | |||
) |
STATISTIC | ( | NumInstrumentedWrites | , |
"Number of instrumented writes" | |||
) |
STATISTIC | ( | NumOfMemProfAllocContextProfiles | , |
"Number of alloc contexts in memory profile." | |||
) |
STATISTIC | ( | NumOfMemProfCallSiteProfiles | , |
"Number of callsites in memory profile." | |||
) |
STATISTIC | ( | NumOfMemProfFunc | , |
"Number of functions having valid memory profile." | |||
) |
STATISTIC | ( | NumOfMemProfMatchedAllocContexts | , |
"Number of matched memory profile alloc contexts." | |||
) |
STATISTIC | ( | NumOfMemProfMatchedAllocs | , |
"Number of matched memory profile allocs." | |||
) |
STATISTIC | ( | NumOfMemProfMatchedCallSites | , |
"Number of matched memory profile callsites." | |||
) |
STATISTIC | ( | NumOfMemProfMismatch | , |
"Number of functions having mismatched memory profile hash." | |||
) |
STATISTIC | ( | NumOfMemProfMissing | , |
"Number of functions without memory profile." | |||
) |
STATISTIC | ( | NumSkippedStackReads | , |
"Number of non-instrumented stack reads" | |||
) |
STATISTIC | ( | NumSkippedStackWrites | , |
"Number of non-instrumented stack writes" | |||
) |
|
static |
Definition at line 922 of file MemProfiler.cpp.
References llvm::memprof::MemProfRecord::AllocSites, llvm::memprof::MemProfRecord::CallSites, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), F, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), and I.
Referenced by readMemprof().
|
static |
|
static |
|
static |
|
static |
|
static |
Referenced by createMemprofHistogramFlagVar(), and llvm::MemProfilerPass::run().
|
static |
|
static |
|
static |
|
static |
|
static |
Referenced by llvm::MemProfilerPass::run().
|
static |
|
static |
|
static |
Referenced by isAllocationWithHotColdVariant().
|
static |
Referenced by readMemprof(), and llvm::MemProfUsePass::run().
|
static |
|
static |
|
constexpr |
Definition at line 64 of file MemProfiler.cpp.
Referenced by llvm::MemProfilerPass::run().
|
constexpr |
Definition at line 70 of file MemProfiler.cpp.
|
constexpr |
Definition at line 67 of file MemProfiler.cpp.
|
constexpr |
Definition at line 61 of file MemProfiler.cpp.
|
constexpr |
Definition at line 73 of file MemProfiler.cpp.
|
constexpr |
Definition at line 75 of file MemProfiler.cpp.
|
constexpr |
Definition at line 83 of file MemProfiler.cpp.
Referenced by createProfileFileNameVar().
|
constexpr |
Definition at line 85 of file MemProfiler.cpp.
Referenced by createMemprofHistogramFlagVar().
|
constexpr |
Definition at line 76 of file MemProfiler.cpp.
|
constexpr |
Definition at line 72 of file MemProfiler.cpp.
|
static |
Referenced by createMemprofDefaultOptionsVar().
|
constexpr |
Definition at line 80 of file MemProfiler.cpp.
|
constexpr |
Definition at line 77 of file MemProfiler.cpp.
|
static |
Referenced by readMemprof().
|
static |
Referenced by readMemprof(), llvm::MemProfUsePass::run(), and llvm::SampleProfileMatcher::runOnModule().