43#define DEBUG_TYPE "memprof"
48template <
class T = u
int64_t>
inline T alignedRead(
const char *
Ptr) {
49 static_assert(std::is_pod<T>::value,
"Not a pod type.");
50 assert(
reinterpret_cast<size_t>(
Ptr) %
sizeof(
T) == 0 &&
"Unaligned Read");
51 return *
reinterpret_cast<const T *
>(
Ptr);
54Error checkBuffer(
const MemoryBuffer &Buffer) {
58 if (Buffer.getBufferSize() == 0)
61 if (Buffer.getBufferSize() <
sizeof(Header)) {
68 const char *Next = Buffer.getBufferStart();
69 while (Next < Buffer.getBufferEnd()) {
70 auto *
H =
reinterpret_cast<const Header *
>(Next);
71 if (
H->Version != MEMPROF_RAW_VERSION) {
75 TotalSize +=
H->TotalSize;
79 if (Buffer.getBufferSize() != TotalSize) {
86 using namespace support;
89 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
Ptr);
92 Items.
push_back(*
reinterpret_cast<const SegmentEntry *
>(
93 Ptr +
I *
sizeof(SegmentEntry)));
99readMemInfoBlocks(
const char *
Ptr) {
100 using namespace support;
103 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
Ptr);
107 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
Ptr);
108 const MemInfoBlock MIB = *
reinterpret_cast<const MemInfoBlock *
>(
Ptr);
111 Ptr +=
sizeof(MemInfoBlock);
117 using namespace support;
120 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
Ptr);
125 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
Ptr);
127 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
Ptr);
129 SmallVector<uint64_t> CallStack;
130 CallStack.reserve(NumPCs);
131 for (
uint64_t J = 0; J < NumPCs; J++) {
133 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
Ptr));
136 Items[StackId] = CallStack;
145 for (
const auto &IdStack :
From) {
146 auto I = To.find(IdStack.first);
148 To[IdStack.first] = IdStack.second;
151 if (IdStack.second !=
I->second)
158Error report(Error
E,
const StringRef
Context) {
163bool isRuntimePath(
const StringRef Path) {
167 return Filename.equals(
"memprof_malloc_linux.cpp") ||
168 Filename.equals(
"memprof_interceptors.cpp") ||
169 Filename.equals(
"memprof_new_delete.cpp");
172std::string getBuildIdString(
const SegmentEntry &Entry) {
174 if (Entry.BuildIdSize == 0)
178 raw_string_ostream
OS(Str);
179 for (
size_t I = 0;
I < Entry.BuildIdSize;
I++) {
186Expected<std::unique_ptr<RawMemProfReader>>
190 if (std::error_code EC = BufferOr.getError())
193 std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
194 return create(std::move(Buffer), ProfiledBinary, KeepName);
199 const StringRef ProfiledBinary,
bool KeepName) {
200 if (
Error E = checkBuffer(*Buffer))
201 return report(std::move(
E), Buffer->getBufferIdentifier());
203 if (ProfiledBinary.
empty()) {
205 const std::vector<std::string> BuildIds =
peekBuildIds(Buffer.get());
206 std::string ErrorMessage(
207 R
"(Path to profiled binary is empty, expected binary with one of the following build ids:
209 for (
const auto &Id : BuildIds) {
210 ErrorMessage +=
"\n BuildId: ";
220 return report(BinaryOr.takeError(), ProfiledBinary);
224 std::unique_ptr<RawMemProfReader> Reader(
226 if (
Error E = Reader->initialize(std::move(Buffer))) {
229 return std::move(Reader);
237 std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
247 return Magic == MEMPROF_RAW_MAGIC_64;
251 uint64_t NumAllocFunctions = 0, NumMibInfo = 0;
253 const size_t NumAllocSites = KV.second.AllocSites.size();
254 if (NumAllocSites > 0) {
256 NumMibInfo += NumAllocSites;
260 OS <<
"MemprofProfile:\n";
262 OS <<
" Version: " << MEMPROF_RAW_VERSION <<
"\n";
263 OS <<
" NumSegments: " << SegmentInfo.
size() <<
"\n";
264 OS <<
" NumMibInfo: " << NumMibInfo <<
"\n";
265 OS <<
" NumAllocFunctions: " << NumAllocFunctions <<
"\n";
266 OS <<
" NumStackOffsets: " << StackMap.
size() <<
"\n";
268 OS <<
" Segments:\n";
269 for (
const auto &Entry : SegmentInfo) {
271 OS <<
" BuildId: " << getBuildIdString(Entry) <<
"\n";
272 OS <<
" Start: 0x" << llvm::utohexstr(Entry.Start) <<
"\n";
273 OS <<
" End: 0x" << llvm::utohexstr(Entry.End) <<
"\n";
274 OS <<
" Offset: 0x" << llvm::utohexstr(Entry.Offset) <<
"\n";
278 for (
const auto &Entry : *
this) {
280 OS <<
" FunctionGUID: " << Entry.first <<
"\n";
281 Entry.second.print(
OS);
285Error RawMemProfReader::initialize(std::unique_ptr<MemoryBuffer> DataBuffer) {
286 const StringRef FileName = Binary.getBinary()->getFileName();
288 auto *ElfObject = dyn_cast<object::ELFObjectFileBase>(Binary.getBinary());
290 return report(make_error<StringError>(
Twine(
"Not an ELF file: "),
298 auto* Elf64LEObject = llvm::cast<llvm::object::ELF64LEObjectFile>(ElfObject);
300 auto PHdrsOr = ElfFile.program_headers();
303 make_error<StringError>(
Twine(
"Could not read program headers: "),
307 int NumExecutableSegments = 0;
308 for (
const auto &Phdr : *PHdrsOr) {
313 if (++NumExecutableSegments > 1) {
315 make_error<StringError>(
316 "Expect only one executable load segment in the binary",
325 PreferredTextSegmentAddress = Phdr.p_vaddr;
326 assert(Phdr.p_vaddr == (Phdr.p_vaddr & ~(0x1000 - 1U)) &&
327 "Expect p_vaddr to always be page aligned");
328 assert(Phdr.p_offset == 0 &&
"Expect p_offset = 0 for symbolization.");
333 auto Triple = ElfObject->makeTriple();
335 return report(make_error<StringError>(Twine(
"Unsupported target: ") +
336 Triple.getArchName(),
341 if (Error
E = readRawProfile(std::move(DataBuffer)))
344 if (Error
E = setupForSymbolization())
347 auto *
Object = cast<object::ObjectFile>(Binary.getBinary());
352 Object, std::move(Context),
false);
354 return report(SOFOr.takeError(), FileName);
355 auto Symbolizer = std::move(SOFOr.get());
361 if (Error
E = symbolizeAndFilterStackFrames(std::move(Symbolizer)))
364 return mapRawProfileToRecords();
367Error RawMemProfReader::setupForSymbolization() {
368 auto *
Object = cast<object::ObjectFile>(Binary.getBinary());
370 if (BinaryId.empty())
371 return make_error<StringError>(Twine(
"No build id found in binary ") +
372 Binary.getBinary()->getFileName(),
376 for (
const auto &Entry : SegmentInfo) {
378 if (BinaryId == SegmentId) {
381 if (++NumMatched > 1) {
382 return make_error<StringError>(
383 "We expect only one executable segment in the profiled binary",
386 ProfiledTextSegmentStart = Entry.Start;
387 ProfiledTextSegmentEnd = Entry.End;
390 assert(NumMatched != 0 &&
"No matching executable segments in segment info.");
391 assert((PreferredTextSegmentAddress == 0 ||
392 (PreferredTextSegmentAddress == ProfiledTextSegmentStart)) &&
393 "Expect text segment address to be 0 or equal to profiled text "
398Error RawMemProfReader::mapRawProfileToRecords() {
404 PerFunctionCallSites;
408 for (
const auto &Entry : CallstackProfileData) {
409 const uint64_t StackId = Entry.first;
411 auto It = StackMap.
find(StackId);
412 if (It == StackMap.
end())
413 return make_error<InstrProfError>(
415 "memprof callstack record does not contain id: " + Twine(StackId));
419 Callstack.
reserve(It->getSecond().size());
422 for (
size_t I = 0;
I < Addresses.
size();
I++) {
425 "Address not found in SymbolizedFrame map");
426 const SmallVector<FrameId> &Frames = SymbolizedFrame[
Address];
429 "The last frame should not be inlined");
434 for (
size_t J = 0; J < Frames.size(); J++) {
435 if (
I == 0 && J == 0)
442 PerFunctionCallSites[Guid].
insert(&Frames);
446 Callstack.
append(Frames.begin(), Frames.end());
453 for (
size_t I = 0; ;
I++) {
457 IndexedMemProfRecord &Record =
Result.first->second;
458 Record.AllocSites.emplace_back(Callstack, CSId, Entry.second);
460 if (!
F.IsInlineFrame)
466 for (
const auto &[Id, Locs] : PerFunctionCallSites) {
470 IndexedMemProfRecord &Record =
Result.first->second;
471 for (LocationPtr Loc : Locs) {
472 Record.CallSites.push_back(*Loc);
481Error RawMemProfReader::symbolizeAndFilterStackFrames(
482 std::unique_ptr<llvm::symbolize::SymbolizableModule> Symbolizer) {
484 const DILineInfoSpecifier Specifier(
485 DILineInfoSpecifier::FileLineInfoKind::RawValue,
486 DILineInfoSpecifier::FunctionNameKind::LinkageName);
494 for (
auto &Entry : StackMap) {
495 for (
const uint64_t VAddr : Entry.getSecond()) {
499 if (SymbolizedFrame.count(VAddr) > 0 ||
503 Expected<DIInliningInfo> DIOr = Symbolizer->symbolizeInlinedCode(
504 getModuleOffset(VAddr), Specifier,
false);
506 return DIOr.takeError();
507 DIInliningInfo DI = DIOr.get();
511 isRuntimePath(DI.getFrame(0).FileName)) {
512 AllVAddrsToDiscard.
insert(VAddr);
516 for (
size_t I = 0, NumFrames = DI.getNumberOfFrames();
I < NumFrames;
518 const auto &DIFrame = DI.getFrame(
I);
521 const Frame
F(Guid, DIFrame.Line - DIFrame.StartLine, DIFrame.Column,
528 if (KeepSymbolName) {
529 StringRef CanonicalName =
531 DIFrame.FunctionName);
532 GuidToSymbolName.
insert({Guid, CanonicalName.str()});
537 SymbolizedFrame[VAddr].push_back(Hash);
541 auto &CallStack = Entry.getSecond();
545 if (CallStack.empty())
546 EntriesToErase.
push_back(Entry.getFirst());
550 for (
const uint64_t Id : EntriesToErase) {
552 CallstackProfileData.erase(Id);
555 if (StackMap.empty())
556 return make_error<InstrProfError>(
558 "no entries in callstack map after symbolization");
563std::vector<std::string>
572 std::vector<std::string> BuildIds;
574 while (Next < DataBuffer->getBufferEnd()) {
575 auto *Header =
reinterpret_cast<const memprof::Header *
>(Next);
578 readSegmentEntries(Next + Header->SegmentOffset);
580 for (
const auto &Entry : Entries) {
581 const std::string Id = getBuildIdString(Entry);
584 BuildIds.push_back(Id);
588 Next += Header->TotalSize;
593Error RawMemProfReader::readRawProfile(
594 std::unique_ptr<MemoryBuffer> DataBuffer) {
595 const char *Next = DataBuffer->getBufferStart();
597 while (Next < DataBuffer->getBufferEnd()) {
598 auto *Header =
reinterpret_cast<const memprof::Header *
>(Next);
603 readSegmentEntries(Next + Header->SegmentOffset);
604 if (!SegmentInfo.empty() && SegmentInfo != Entries) {
608 return make_error<InstrProfError>(
610 "memprof raw profile has different segment information");
612 SegmentInfo.assign(Entries.begin(), Entries.end());
617 for (
const auto &
Value : readMemInfoBlocks(Next + Header->MIBOffset)) {
618 if (CallstackProfileData.count(
Value.first)) {
619 CallstackProfileData[
Value.first].Merge(
Value.second);
621 CallstackProfileData[Value.first] = Value.second;
627 const CallStackMap CSM = readStackInfo(Next + Header->StackOffset);
628 if (StackMap.empty()) {
631 if (mergeStackMap(CSM, StackMap))
632 return make_error<InstrProfError>(
634 "memprof raw profile got different call stack for same id");
637 Next += Header->TotalSize;
643object::SectionedAddress
644RawMemProfReader::getModuleOffset(
const uint64_t VirtualAddress) {
645 if (VirtualAddress > ProfiledTextSegmentStart &&
646 VirtualAddress <= ProfiledTextSegmentEnd) {
652 VirtualAddress + PreferredTextSegmentAddress - ProfiledTextSegmentStart;
653 return object::SectionedAddress{AdjustedAddress};
658 return object::SectionedAddress{VirtualAddress};
667 auto IdToFrameCallback = [
this](
const FrameId Id) {
669 if (!this->KeepSymbolName)
671 auto Iter = this->GuidToSymbolName.
find(
F.Function);
673 F.SymbolName =
Iter->getSecond();
BlockVerifier::State From
This file declares a library for handling Build IDs and using them to find debug info.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file defines the DenseMap class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallSet class.
This file defines the SmallVector class.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
static std::unique_ptr< DWARFContext > create(const object::ObjectFile &Obj, ProcessDebugRelocations RelocAction=ProcessDebugRelocations::Process, const LoadedObjectInfo *L=nullptr, std::string DWPName="", std::function< void(Error)> RecoverableErrorHandler=WithColor::defaultErrorHandler, std::function< void(Error)> WarningHandler=WithColor::defaultWarningHandler, bool ThreadSafe=false)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
This class implements a map that also provides access to all stored values in a deterministic order.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This interface provides simple read-only access to a block of memory, and provides simple methods for...
size_t getBufferSize() const
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
const char * getBufferStart() const
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
bool contains(const T &V) const
Check if the SmallSet contains the given element.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
const Frame & idToFrame(const FrameId Id) const
virtual Error readNextRecord(GuidMemProfRecordPair &GuidRecord, std::function< const Frame(const FrameId)> Callback=nullptr)
llvm::DenseMap< FrameId, Frame > IdToFrame
llvm::MapVector< GlobalValue::GUID, IndexedMemProfRecord >::iterator Iter
llvm::MapVector< GlobalValue::GUID, IndexedMemProfRecord > FunctionProfileData
std::pair< GlobalValue::GUID, MemProfRecord > GuidMemProfRecordPair
void printYAML(raw_ostream &OS)
static Expected< std::unique_ptr< RawMemProfReader > > create(const Twine &Path, StringRef ProfiledBinary, bool KeepName=false)
static std::vector< std::string > peekBuildIds(MemoryBuffer *DataBuffer)
virtual Error readNextRecord(GuidMemProfRecordPair &GuidRecord, std::function< const Frame(const FrameId)> Callback) override
static bool hasFormat(const MemoryBuffer &DataBuffer)
This class implements an extremely fast bulk output stream that can only output to a stream.
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
static Expected< std::unique_ptr< SymbolizableObjectFile > > create(const object::ObjectFile *Obj, std::unique_ptr< DIContext > DICtx, bool UntagAddresses)
CallStackId hashCallStack(ArrayRef< FrameId > CS)
llvm::DenseMap< uint64_t, llvm::SmallVector< uint64_t > > CallStackMap
void verifyFunctionProfileData(const llvm::MapVector< GlobalValue::GUID, IndexedMemProfRecord > &FunctionProfileData)
BuildIDRef getBuildID(const ObjectFile *Obj)
Returns the build ID, if any, contained in the given object file.
ArrayRef< uint8_t > BuildIDRef
A reference to a BuildID in binary form.
Expected< std::unique_ptr< Binary > > createBinary(MemoryBufferRef Source, LLVMContext *Context=nullptr, bool InitContent=true)
Create a Binary from Source, autodetecting the file type.
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
This is an optimization pass for GlobalISel generic memory operations.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
Error joinErrors(Error E1, Error E2)
Concatenate errors.
FormattedNumber format_hex_no_prefix(uint64_t N, unsigned Width, bool Upper=false)
format_hex_no_prefix - Output N as a fixed width hexadecimal.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
static constexpr const char *const BadString
GlobalValue::GUID Function
static GlobalValue::GUID getGUID(const StringRef FunctionName)