16 #include <type_traits>
35 #define DEBUG_TYPE "memprof"
40 template <
class T = u
int64_t>
inline T alignedRead(
const char *Ptr) {
41 static_assert(std::is_pod<T>::value,
"Not a pod type.");
42 assert(
reinterpret_cast<size_t>(Ptr) %
sizeof(
T) == 0 &&
"Unaligned Read");
43 return *
reinterpret_cast<const T *
>(Ptr);
46 Error checkBuffer(
const MemoryBuffer &Buffer) {
50 if (Buffer.getBufferSize() == 0)
53 if (Buffer.getBufferSize() <
sizeof(Header)) {
60 const char *Next = Buffer.getBufferStart();
61 while (Next < Buffer.getBufferEnd()) {
62 auto *
H =
reinterpret_cast<const Header *
>(Next);
63 if (
H->Version != MEMPROF_RAW_VERSION) {
67 TotalSize +=
H->TotalSize;
71 if (Buffer.getBufferSize() != TotalSize) {
81 endian::readNext<uint64_t, little, unaligned>(Ptr);
84 Items.push_back(*
reinterpret_cast<const SegmentEntry *
>(
85 Ptr +
I *
sizeof(SegmentEntry)));
91 readMemInfoBlocks(
const char *Ptr) {
95 endian::readNext<uint64_t, little, unaligned>(Ptr);
98 const uint64_t Id = endian::readNext<uint64_t, little, unaligned>(Ptr);
99 const MemInfoBlock MIB = *
reinterpret_cast<const MemInfoBlock *
>(Ptr);
100 Items.push_back({
Id, MIB});
102 Ptr +=
sizeof(MemInfoBlock);
111 endian::readNext<uint64_t, little, unaligned>(Ptr);
115 const uint64_t StackId = endian::readNext<uint64_t, little, unaligned>(Ptr);
116 const uint64_t NumPCs = endian::readNext<uint64_t, little, unaligned>(Ptr);
118 SmallVector<uint64_t> CallStack;
119 for (
uint64_t J = 0; J < NumPCs; J++) {
120 CallStack.push_back(endian::readNext<uint64_t, little, unaligned>(Ptr));
123 Items[StackId] = CallStack;
132 for (
const auto &IdStack :
From) {
133 auto I = To.find(IdStack.first);
135 To[IdStack.first] = IdStack.second;
138 if (IdStack.second !=
I->second)
145 Error report(Error
E,
const StringRef
Context) {
150 bool isRuntimePath(
const StringRef Path) {
152 .contains(
"memprof/memprof_");
155 std::string getBuildIdString(
const SegmentEntry &Entry) {
156 constexpr
size_t Size =
sizeof(Entry.BuildId) /
sizeof(uint8_t);
157 constexpr uint8_t Zeros[Size] = {0};
159 if (
memcmp(Entry.BuildId, Zeros, Size) == 0)
163 raw_string_ostream OS(Str);
164 for (
size_t I = 0;
I < Size;
I++) {
171 Expected<std::unique_ptr<RawMemProfReader>>
175 if (std::error_code EC = BufferOr.getError())
178 std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
179 if (
Error E = checkBuffer(*Buffer))
180 return report(
std::move(
E), Path.getSingleStringRef());
182 if (ProfiledBinary.
empty())
185 "Path to profiled binary is empty!");
189 return report(BinaryOr.takeError(), ProfiledBinary);
193 std::unique_ptr<RawMemProfReader> Reader(
206 std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
216 return Magic == MEMPROF_RAW_MAGIC_64;
220 uint64_t NumAllocFunctions = 0, NumMibInfo = 0;
221 for (
const auto &KV : FunctionProfileData) {
222 const size_t NumAllocSites = KV.second.AllocSites.size();
223 if (NumAllocSites > 0) {
225 NumMibInfo += NumAllocSites;
229 OS <<
"MemprofProfile:\n";
231 OS <<
" Version: " << MEMPROF_RAW_VERSION <<
"\n";
232 OS <<
" NumSegments: " << SegmentInfo.size() <<
"\n";
233 OS <<
" NumMibInfo: " << NumMibInfo <<
"\n";
234 OS <<
" NumAllocFunctions: " << NumAllocFunctions <<
"\n";
235 OS <<
" NumStackOffsets: " << StackMap.
size() <<
"\n";
237 OS <<
" Segments:\n";
238 for (
const auto &Entry : SegmentInfo) {
240 OS <<
" BuildId: " << getBuildIdString(Entry) <<
"\n";
241 OS <<
" Start: 0x" << llvm::utohexstr(Entry.Start) <<
"\n";
242 OS <<
" End: 0x" << llvm::utohexstr(Entry.End) <<
"\n";
243 OS <<
" Offset: 0x" << llvm::utohexstr(Entry.Offset) <<
"\n";
247 for (
const auto &Entry : *
this) {
249 OS <<
" FunctionGUID: " << Entry.first <<
"\n";
250 Entry.second.print(OS);
254 Error RawMemProfReader::initialize(std::unique_ptr<MemoryBuffer> DataBuffer) {
255 const StringRef FileName = Binary.getBinary()->getFileName();
257 auto *ElfObject = dyn_cast<object::ELFObjectFileBase>(Binary.getBinary());
259 return report(make_error<StringError>(
Twine(
"Not an ELF file: "),
264 auto Triple = ElfObject->makeTriple();
266 return report(make_error<StringError>(
Twine(
"Unsupported target: ") +
271 auto *
Object = cast<object::ObjectFile>(Binary.getBinary());
278 return report(SOFOr.takeError(), FileName);
284 if (
Error E = symbolizeAndFilterStackFrames())
287 return mapRawProfileToRecords();
290 Error RawMemProfReader::mapRawProfileToRecords() {
296 PerFunctionCallSites;
300 for (
const auto &Entry : CallstackProfileData) {
301 const uint64_t StackId = Entry.first;
303 auto It = StackMap.
find(StackId);
304 if (It == StackMap.
end())
305 return make_error<InstrProfError>(
307 "memprof callstack record does not contain id: " + Twine(StackId));
311 Callstack.
reserve(It->getSecond().size());
314 for (
size_t I = 0;
I < Addresses.size();
I++) {
316 assert(SymbolizedFrame.count(Address) > 0 &&
317 "Address not found in SymbolizedFrame map");
318 const SmallVector<FrameId> &Frames = SymbolizedFrame[Address];
321 "The last frame should not be inlined");
326 for (
size_t J = 0; J < Frames.size(); J++) {
327 if (
I == 0 && J == 0)
334 PerFunctionCallSites[Guid].
insert(&Frames);
338 Callstack.
append(Frames.begin(), Frames.end());
343 for (
size_t I = 0; ;
I++) {
344 const Frame &
F = idToFrame(Callstack[
I]);
346 FunctionProfileData.insert({
F.Function, IndexedMemProfRecord()});
347 IndexedMemProfRecord &Record =
Result.first->second;
348 Record.AllocSites.emplace_back(Callstack, Entry.second);
350 if (!
F.IsInlineFrame)
356 for (
auto I = PerFunctionCallSites.
begin(),
E = PerFunctionCallSites.
end();
361 auto Result = FunctionProfileData.insert({
Id, IndexedMemProfRecord()});
362 IndexedMemProfRecord &Record =
Result.first->second;
363 for (LocationPtr Loc :
I->getSecond()) {
364 Record.CallSites.push_back(*Loc);
371 Error RawMemProfReader::symbolizeAndFilterStackFrames() {
373 const DILineInfoSpecifier Specifier(
383 for (
auto &Entry : StackMap) {
384 for (
const uint64_t VAddr : Entry.getSecond()) {
388 if (SymbolizedFrame.count(VAddr) > 0 ||
392 Expected<DIInliningInfo> DIOr = Symbolizer->symbolizeInlinedCode(
393 getModuleOffset(VAddr), Specifier,
false);
395 return DIOr.takeError();
396 DIInliningInfo DI = DIOr.get();
400 isRuntimePath(DI.getFrame(0).FileName)) {
401 AllVAddrsToDiscard.
insert(VAddr);
405 for (
size_t I = 0, NumFrames = DI.getNumberOfFrames();
I < NumFrames;
407 const auto &DIFrame = DI.getFrame(
I);
410 const Frame
F(Guid, DIFrame.Line - DIFrame.StartLine, DIFrame.Column,
418 GuidToSymbolName.
insert({Guid, DIFrame.FunctionName});
421 IdToFrame.insert({Hash,
F});
422 SymbolizedFrame[VAddr].push_back(Hash);
426 auto &CallStack = Entry.getSecond();
428 return AllVAddrsToDiscard.
contains(A);
430 if (CallStack.empty())
431 EntriesToErase.push_back(Entry.getFirst());
437 CallstackProfileData.erase(
Id);
440 if (StackMap.empty())
441 return make_error<InstrProfError>(
443 "no entries in callstack map after symbolization");
448 Error RawMemProfReader::readRawProfile(
449 std::unique_ptr<MemoryBuffer> DataBuffer) {
450 const char *Next = DataBuffer->getBufferStart();
452 while (Next < DataBuffer->getBufferEnd()) {
453 auto *Header =
reinterpret_cast<const memprof::Header *
>(Next);
458 readSegmentEntries(Next + Header->SegmentOffset);
459 if (!SegmentInfo.empty() && SegmentInfo != Entries) {
463 return make_error<InstrProfError>(
465 "memprof raw profile has different segment information");
467 SegmentInfo.
assign(Entries.begin(), Entries.end());
472 for (
const auto &
Value : readMemInfoBlocks(Next + Header->MIBOffset)) {
473 if (CallstackProfileData.count(
Value.first)) {
474 CallstackProfileData[
Value.first].Merge(
Value.second);
476 CallstackProfileData[
Value.first] =
Value.second;
482 const CallStackMap CSM = readStackInfo(Next + Header->StackOffset);
483 if (StackMap.empty()) {
486 if (mergeStackMap(CSM, StackMap))
487 return make_error<InstrProfError>(
489 "memprof raw profile got different call stack for same id");
492 Next += Header->TotalSize;
498 object::SectionedAddress
499 RawMemProfReader::getModuleOffset(
const uint64_t VirtualAddress) {
501 SegmentEntry *ContainingSegment =
nullptr;
502 for (
auto &SE : SegmentInfo) {
503 if (VirtualAddress > SE.Start && VirtualAddress <= SE.End) {
504 ContainingSegment = &SE;
509 assert(ContainingSegment &&
"Could not find a segment entry");
514 return object::SectionedAddress{VirtualAddress};
518 if (FunctionProfileData.empty())
521 if (Iter == FunctionProfileData.end())
524 auto IdToFrameCallback = [
this](
const FrameId Id) {
526 if (!this->KeepSymbolName)
528 auto Iter = this->GuidToSymbolName.
find(
F.Function);
529 assert(Iter != this->GuidToSymbolName.
end());
530 F.SymbolName = Iter->getSecond();
535 GuidRecord = {Iter->first,
MemProfRecord(IndexedRecord, IdToFrameCallback)};