LLVM 19.0.0git
InstrProfReader.cpp
Go to the documentation of this file.
1//===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for reading profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/StringRef.h"
24#include "llvm/Support/Endian.h"
25#include "llvm/Support/Error.h"
30#include <algorithm>
31#include <cstddef>
32#include <cstdint>
33#include <limits>
34#include <memory>
35#include <system_error>
36#include <utility>
37#include <vector>
38
39using namespace llvm;
40
41// Extracts the variant information from the top 32 bits in the version and
42// returns an enum specifying the variants present.
44 InstrProfKind ProfileKind = InstrProfKind::Unknown;
45 if (Version & VARIANT_MASK_IR_PROF) {
46 ProfileKind |= InstrProfKind::IRInstrumentation;
47 }
48 if (Version & VARIANT_MASK_CSIR_PROF) {
49 ProfileKind |= InstrProfKind::ContextSensitive;
50 }
51 if (Version & VARIANT_MASK_INSTR_ENTRY) {
52 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
53 }
54 if (Version & VARIANT_MASK_BYTE_COVERAGE) {
55 ProfileKind |= InstrProfKind::SingleByteCoverage;
56 }
57 if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
58 ProfileKind |= InstrProfKind::FunctionEntryOnly;
59 }
60 if (Version & VARIANT_MASK_MEMPROF) {
61 ProfileKind |= InstrProfKind::MemProf;
62 }
63 if (Version & VARIANT_MASK_TEMPORAL_PROF) {
64 ProfileKind |= InstrProfKind::TemporalProfile;
65 }
66 return ProfileKind;
67}
68
71 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
72 : FS.getBufferForFile(Filename);
73 if (std::error_code EC = BufferOrErr.getError())
74 return errorCodeToError(EC);
75 return std::move(BufferOrErr.get());
76}
77
79 return Reader.readHeader();
80}
81
82/// Read a list of binary ids from a profile that consist of
83/// a. uint64_t binary id length
84/// b. uint8_t binary id data
85/// c. uint8_t padding (if necessary)
86/// This function is shared between raw and indexed profiles.
87/// Raw profiles are in host-endian format, and indexed profiles are in
88/// little-endian format. So, this function takes an argument indicating the
89/// associated endian format to read the binary ids correctly.
90static Error
92 const uint64_t BinaryIdsSize,
93 const uint8_t *BinaryIdsStart,
94 std::vector<llvm::object::BuildID> &BinaryIds,
95 const llvm::endianness Endian) {
96 using namespace support;
97
98 if (BinaryIdsSize == 0)
99 return Error::success();
100
101 const uint8_t *BI = BinaryIdsStart;
102 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
103 const uint8_t *End =
104 reinterpret_cast<const uint8_t *>(DataBuffer.getBufferEnd());
105
106 while (BI < BIEnd) {
107 size_t Remaining = BIEnd - BI;
108 // There should be enough left to read the binary id length.
109 if (Remaining < sizeof(uint64_t))
110 return make_error<InstrProfError>(
111 instrprof_error::malformed,
112 "not enough data to read binary id length");
113
114 uint64_t BILen = 0;
116 BILen =
117 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(BI);
118 else
119 BILen = endian::readNext<uint64_t, llvm::endianness::big, unaligned>(BI);
120
121 if (BILen == 0)
122 return make_error<InstrProfError>(instrprof_error::malformed,
123 "binary id length is 0");
124
125 Remaining = BIEnd - BI;
126 // There should be enough left to read the binary id data.
127 if (Remaining < alignToPowerOf2(BILen, sizeof(uint64_t)))
128 return make_error<InstrProfError>(
129 instrprof_error::malformed, "not enough data to read binary id data");
130
131 // Add binary id to the binary ids list.
132 BinaryIds.push_back(object::BuildID(BI, BI + BILen));
133
134 // Increment by binary id data length, which aligned to the size of uint64.
135 BI += alignToPowerOf2(BILen, sizeof(uint64_t));
136 if (BI > End)
137 return make_error<InstrProfError>(
138 instrprof_error::malformed,
139 "binary id section is greater than buffer size");
140 }
141
142 return Error::success();
143}
144
145static void
147 std::vector<llvm::object::BuildID> &BinaryIds) {
148 OS << "Binary IDs: \n";
149 for (auto BI : BinaryIds) {
150 for (uint64_t I = 0; I < BI.size(); I++)
151 OS << format("%02x", BI[I]);
152 OS << "\n";
153 }
154}
155
158 const InstrProfCorrelator *Correlator,
159 std::function<void(Error)> Warn) {
160 // Set up the buffer to read.
161 auto BufferOrError = setupMemoryBuffer(Path, FS);
162 if (Error E = BufferOrError.takeError())
163 return std::move(E);
164 return InstrProfReader::create(std::move(BufferOrError.get()), Correlator,
165 Warn);
166}
167
169InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
170 const InstrProfCorrelator *Correlator,
171 std::function<void(Error)> Warn) {
172 if (Buffer->getBufferSize() == 0)
173 return make_error<InstrProfError>(instrprof_error::empty_raw_profile);
174
175 std::unique_ptr<InstrProfReader> Result;
176 // Create the reader.
178 Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
179 else if (RawInstrProfReader64::hasFormat(*Buffer))
180 Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator, Warn));
181 else if (RawInstrProfReader32::hasFormat(*Buffer))
182 Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator, Warn));
183 else if (TextInstrProfReader::hasFormat(*Buffer))
184 Result.reset(new TextInstrProfReader(std::move(Buffer)));
185 else
186 return make_error<InstrProfError>(instrprof_error::unrecognized_format);
187
188 // Initialize the reader and return the result.
189 if (Error E = initializeReader(*Result))
190 return std::move(E);
191
192 return std::move(Result);
193}
194
197 const Twine &RemappingPath) {
198 // Set up the buffer to read.
199 auto BufferOrError = setupMemoryBuffer(Path, FS);
200 if (Error E = BufferOrError.takeError())
201 return std::move(E);
202
203 // Set up the remapping buffer if requested.
204 std::unique_ptr<MemoryBuffer> RemappingBuffer;
205 std::string RemappingPathStr = RemappingPath.str();
206 if (!RemappingPathStr.empty()) {
207 auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr, FS);
208 if (Error E = RemappingBufferOrError.takeError())
209 return std::move(E);
210 RemappingBuffer = std::move(RemappingBufferOrError.get());
211 }
212
213 return IndexedInstrProfReader::create(std::move(BufferOrError.get()),
214 std::move(RemappingBuffer));
215}
216
218IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
219 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
220 // Create the reader.
222 return make_error<InstrProfError>(instrprof_error::bad_magic);
223 auto Result = std::make_unique<IndexedInstrProfReader>(
224 std::move(Buffer), std::move(RemappingBuffer));
225
226 // Initialize the reader and return the result.
227 if (Error E = initializeReader(*Result))
228 return std::move(E);
229
230 return std::move(Result);
231}
232
234 // Verify that this really looks like plain ASCII text by checking a
235 // 'reasonable' number of characters (up to profile magic size).
236 size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t));
237 StringRef buffer = Buffer.getBufferStart();
238 return count == 0 ||
239 std::all_of(buffer.begin(), buffer.begin() + count,
240 [](char c) { return isPrint(c) || isSpace(c); });
241}
242
243// Read the profile variant flag from the header: ":FE" means this is a FE
244// generated profile. ":IR" means this is an IR level profile. Other strings
245// with a leading ':' will be reported an error format.
247 Symtab.reset(new InstrProfSymtab());
248
249 while (Line->starts_with(":")) {
250 StringRef Str = Line->substr(1);
251 if (Str.equals_insensitive("ir"))
253 else if (Str.equals_insensitive("fe"))
255 else if (Str.equals_insensitive("csir")) {
257 ProfileKind |= InstrProfKind::ContextSensitive;
258 } else if (Str.equals_insensitive("entry_first"))
260 else if (Str.equals_insensitive("not_entry_first"))
262 else if (Str.equals_insensitive("single_byte_coverage"))
264 else if (Str.equals_insensitive("temporal_prof_traces")) {
265 ProfileKind |= InstrProfKind::TemporalProfile;
266 if (auto Err = readTemporalProfTraceData())
267 return error(std::move(Err));
268 } else
270 ++Line;
271 }
272 return success();
273}
274
275/// Temporal profile trace data is stored in the header immediately after
276/// ":temporal_prof_traces". The first integer is the number of traces, the
277/// second integer is the stream size, then the following lines are the actual
278/// traces which consist of a weight and a comma separated list of function
279/// names.
280Error TextInstrProfReader::readTemporalProfTraceData() {
281 if ((++Line).is_at_end())
283
284 uint32_t NumTraces;
285 if (Line->getAsInteger(0, NumTraces))
287
288 if ((++Line).is_at_end())
290
293
294 for (uint32_t i = 0; i < NumTraces; i++) {
295 if ((++Line).is_at_end())
297
299 if (Line->getAsInteger(0, Trace.Weight))
301
302 if ((++Line).is_at_end())
304
305 SmallVector<StringRef> FuncNames;
306 Line->split(FuncNames, ",", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
307 for (auto &FuncName : FuncNames)
308 Trace.FunctionNameRefs.push_back(
309 IndexedInstrProf::ComputeHash(FuncName.trim()));
310 TemporalProfTraces.push_back(std::move(Trace));
311 }
312 return success();
313}
314
315Error
316TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
317
318#define CHECK_LINE_END(Line) \
319 if (Line.is_at_end()) \
320 return error(instrprof_error::truncated);
321#define READ_NUM(Str, Dst) \
322 if ((Str).getAsInteger(10, (Dst))) \
323 return error(instrprof_error::malformed);
324#define VP_READ_ADVANCE(Val) \
325 CHECK_LINE_END(Line); \
326 uint32_t Val; \
327 READ_NUM((*Line), (Val)); \
328 Line++;
329
330 if (Line.is_at_end())
331 return success();
332
333 uint32_t NumValueKinds;
334 if (Line->getAsInteger(10, NumValueKinds)) {
335 // No value profile data
336 return success();
337 }
338 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
340 "number of value kinds is invalid");
341 Line++;
342
343 for (uint32_t VK = 0; VK < NumValueKinds; VK++) {
344 VP_READ_ADVANCE(ValueKind);
345 if (ValueKind > IPVK_Last)
346 return error(instrprof_error::malformed, "value kind is invalid");
347 ;
348 VP_READ_ADVANCE(NumValueSites);
349 if (!NumValueSites)
350 continue;
351
352 Record.reserveSites(VK, NumValueSites);
353 for (uint32_t S = 0; S < NumValueSites; S++) {
354 VP_READ_ADVANCE(NumValueData);
355
356 std::vector<InstrProfValueData> CurrentValues;
357 for (uint32_t V = 0; V < NumValueData; V++) {
358 CHECK_LINE_END(Line);
359 std::pair<StringRef, StringRef> VD = Line->rsplit(':');
360 uint64_t TakenCount, Value;
361 if (ValueKind == IPVK_IndirectCallTarget) {
362 if (InstrProfSymtab::isExternalSymbol(VD.first)) {
363 Value = 0;
364 } else {
365 if (Error E = Symtab->addFuncName(VD.first))
366 return E;
368 }
369 } else if (ValueKind == IPVK_VTableTarget) {
371 Value = 0;
372 else
374 } else {
375 READ_NUM(VD.first, Value);
376 }
377 READ_NUM(VD.second, TakenCount);
378 CurrentValues.push_back({Value, TakenCount});
379 Line++;
380 }
381 Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
382 nullptr);
383 }
384 }
385 return success();
386
387#undef CHECK_LINE_END
388#undef READ_NUM
389#undef VP_READ_ADVANCE
390}
391
393 // Skip empty lines and comments.
394 while (!Line.is_at_end() && (Line->empty() || Line->starts_with("#")))
395 ++Line;
396 // If we hit EOF while looking for a name, we're done.
397 if (Line.is_at_end()) {
399 }
400
401 // Read the function name.
402 Record.Name = *Line++;
403 if (Error E = Symtab->addFuncName(Record.Name))
404 return error(std::move(E));
405
406 // Read the function hash.
407 if (Line.is_at_end())
409 if ((Line++)->getAsInteger(0, Record.Hash))
411 "function hash is not a valid integer");
412
413 // Read the number of counters.
414 uint64_t NumCounters;
415 if (Line.is_at_end())
417 if ((Line++)->getAsInteger(10, NumCounters))
419 "number of counters is not a valid integer");
420 if (NumCounters == 0)
421 return error(instrprof_error::malformed, "number of counters is zero");
422
423 // Read each counter and fill our internal storage with the values.
424 Record.Clear();
425 Record.Counts.reserve(NumCounters);
426 for (uint64_t I = 0; I < NumCounters; ++I) {
427 if (Line.is_at_end())
429 uint64_t Count;
430 if ((Line++)->getAsInteger(10, Count))
431 return error(instrprof_error::malformed, "count is invalid");
432 Record.Counts.push_back(Count);
433 }
434
435 // Bitmap byte information is indicated with special character.
436 if (Line->starts_with("$")) {
437 Record.BitmapBytes.clear();
438 // Read the number of bitmap bytes.
439 uint64_t NumBitmapBytes;
440 if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes))
442 "number of bitmap bytes is not a valid integer");
443 if (NumBitmapBytes != 0) {
444 // Read each bitmap and fill our internal storage with the values.
445 Record.BitmapBytes.reserve(NumBitmapBytes);
446 for (uint8_t I = 0; I < NumBitmapBytes; ++I) {
447 if (Line.is_at_end())
449 uint8_t BitmapByte;
450 if ((Line++)->getAsInteger(0, BitmapByte))
452 "bitmap byte is not a valid integer");
453 Record.BitmapBytes.push_back(BitmapByte);
454 }
455 }
456 }
457
458 // Check if value profile data exists and read it if so.
459 if (Error E = readValueProfileData(Record))
460 return error(std::move(E));
461
462 return success();
463}
464
465template <class IntPtrT>
467 return getProfileKindFromVersion(Version);
468}
469
470template <class IntPtrT>
473 std::optional<uint64_t> Weight) {
474 if (TemporalProfTimestamps.empty()) {
475 assert(TemporalProfTraces.empty());
476 return TemporalProfTraces;
477 }
478 // Sort functions by their timestamps to build the trace.
479 std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end());
481 if (Weight)
482 Trace.Weight = *Weight;
483 for (auto &[TimestampValue, NameRef] : TemporalProfTimestamps)
484 Trace.FunctionNameRefs.push_back(NameRef);
485 TemporalProfTraces = {std::move(Trace)};
486 return TemporalProfTraces;
487}
488
489template <class IntPtrT>
491 if (DataBuffer.getBufferSize() < sizeof(uint64_t))
492 return false;
493 uint64_t Magic =
494 *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
495 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
496 llvm::byteswap(RawInstrProf::getMagic<IntPtrT>()) == Magic;
497}
498
499template <class IntPtrT>
501 if (!hasFormat(*DataBuffer))
503 if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
505 auto *Header = reinterpret_cast<const RawInstrProf::Header *>(
506 DataBuffer->getBufferStart());
507 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
508 return readHeader(*Header);
509}
510
511template <class IntPtrT>
513 const char *End = DataBuffer->getBufferEnd();
514 // Skip zero padding between profiles.
515 while (CurrentPos != End && *CurrentPos == 0)
516 ++CurrentPos;
517 // If there's nothing left, we're done.
518 if (CurrentPos == End)
519 return make_error<InstrProfError>(instrprof_error::eof);
520 // If there isn't enough space for another header, this is probably just
521 // garbage at the end of the file.
522 if (CurrentPos + sizeof(RawInstrProf::Header) > End)
523 return make_error<InstrProfError>(instrprof_error::malformed,
524 "not enough space for another header");
525 // The writer ensures each profile is padded to start at an aligned address.
526 if (reinterpret_cast<size_t>(CurrentPos) % alignof(uint64_t))
527 return make_error<InstrProfError>(instrprof_error::malformed,
528 "insufficient padding");
529 // The magic should have the same byte order as in the previous header.
530 uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
531 if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
532 return make_error<InstrProfError>(instrprof_error::bad_magic);
533
534 // There's another profile to read, so we need to process the header.
535 auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
536 return readHeader(*Header);
537}
538
539template <class IntPtrT>
541 if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart)))
542 return error(std::move(E));
543 for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
544 const IntPtrT FPtr = swap(I->FunctionPointer);
545 if (!FPtr)
546 continue;
547 Symtab.mapAddress(FPtr, swap(I->NameRef));
548 }
549 return success();
550}
551
552template <class IntPtrT>
554 const RawInstrProf::Header &Header) {
555 Version = swap(Header.Version);
556 if (GET_VERSION(Version) != RawInstrProf::Version)
558 ("Profile uses raw profile format version = " +
559 Twine(GET_VERSION(Version)) +
560 "; expected version = " + Twine(RawInstrProf::Version) +
561 "\nPLEASE update this tool to version in the raw profile, or "
562 "regenerate raw profile with expected version.")
563 .str());
564
565 uint64_t BinaryIdSize = swap(Header.BinaryIdsSize);
566 // Binary id start just after the header if exists.
567 const uint8_t *BinaryIdStart =
568 reinterpret_cast<const uint8_t *>(&Header) + sizeof(RawInstrProf::Header);
569 const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;
570 const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd();
571 if (BinaryIdSize % sizeof(uint64_t) || BinaryIdEnd > BufferEnd)
573 if (BinaryIdSize != 0) {
574 if (Error Err =
575 readBinaryIdsInternal(*DataBuffer, BinaryIdSize, BinaryIdStart,
576 BinaryIds, getDataEndianness()))
577 return Err;
578 }
579
580 CountersDelta = swap(Header.CountersDelta);
581 BitmapDelta = swap(Header.BitmapDelta);
582 NamesDelta = swap(Header.NamesDelta);
583 auto NumData = swap(Header.NumData);
584 auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters);
585 auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize();
586 auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
587 auto NumBitmapBytes = swap(Header.NumBitmapBytes);
588 auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes);
589 auto NamesSize = swap(Header.NamesSize);
590 auto VTableNameSize = swap(Header.VNamesSize);
591 auto NumVTables = swap(Header.NumVTables);
592 ValueKindLast = swap(Header.ValueKindLast);
593
594 auto DataSize = NumData * sizeof(RawInstrProf::ProfileData<IntPtrT>);
595 auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
596 auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
597
598 auto VTableSectionSize =
599 NumVTables * sizeof(RawInstrProf::VTableProfileData<IntPtrT>);
600 auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
601
602 // Profile data starts after profile header and binary ids if exist.
603 ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdSize;
604 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
605 ptrdiff_t BitmapOffset =
606 CountersOffset + CountersSize + PaddingBytesAfterCounters;
607 ptrdiff_t NamesOffset =
608 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
609 ptrdiff_t VTableProfDataOffset =
610 NamesOffset + NamesSize + PaddingBytesAfterNames;
611 ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
612 PaddingBytesAfterVTableProfData;
613 ptrdiff_t ValueDataOffset =
614 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
615
616 auto *Start = reinterpret_cast<const char *>(&Header);
617 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
619
620 if (Correlator) {
621 // These sizes in the raw file are zero because we constructed them in the
622 // Correlator.
623 if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
624 NamesDelta == 0))
626 Data = Correlator->getDataPointer();
627 DataEnd = Data + Correlator->getDataSize();
628 NamesStart = Correlator->getNamesPointer();
629 NamesEnd = NamesStart + Correlator->getNamesSize();
630 } else {
631 Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
632 Start + DataOffset);
633 DataEnd = Data + NumData;
634 VTableBegin =
635 reinterpret_cast<const RawInstrProf::VTableProfileData<IntPtrT> *>(
636 Start + VTableProfDataOffset);
637 VTableEnd = VTableBegin + NumVTables;
638 NamesStart = Start + NamesOffset;
639 NamesEnd = NamesStart + NamesSize;
640 VNamesStart = Start + VTableNameOffset;
641 VNamesEnd = VNamesStart + VTableNameSize;
642 }
643
644 CountersStart = Start + CountersOffset;
645 CountersEnd = CountersStart + CountersSize;
646 BitmapStart = Start + BitmapOffset;
647 BitmapEnd = BitmapStart + NumBitmapBytes;
648 ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
649
650 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
651 if (Error E = createSymtab(*NewSymtab))
652 return E;
653
654 Symtab = std::move(NewSymtab);
655 return success();
656}
657
658template <class IntPtrT>
660 Record.Name = getName(Data->NameRef);
661 return success();
662}
663
664template <class IntPtrT>
666 Record.Hash = swap(Data->FuncHash);
667 return success();
668}
669
670template <class IntPtrT>
673 uint32_t NumCounters = swap(Data->NumCounters);
674 if (NumCounters == 0)
675 return error(instrprof_error::malformed, "number of counters is zero");
676
677 ptrdiff_t CounterBaseOffset = swap(Data->CounterPtr) - CountersDelta;
678 if (CounterBaseOffset < 0)
679 return error(
681 ("counter offset " + Twine(CounterBaseOffset) + " is negative").str());
682
683 if (CounterBaseOffset >= CountersEnd - CountersStart)
685 ("counter offset " + Twine(CounterBaseOffset) +
686 " is greater than the maximum counter offset " +
687 Twine(CountersEnd - CountersStart - 1))
688 .str());
689
690 uint64_t MaxNumCounters =
691 (CountersEnd - (CountersStart + CounterBaseOffset)) /
692 getCounterTypeSize();
693 if (NumCounters > MaxNumCounters)
695 ("number of counters " + Twine(NumCounters) +
696 " is greater than the maximum number of counters " +
697 Twine(MaxNumCounters))
698 .str());
699
700 Record.Counts.clear();
701 Record.Counts.reserve(NumCounters);
702 for (uint32_t I = 0; I < NumCounters; I++) {
703 const char *Ptr =
704 CountersStart + CounterBaseOffset + I * getCounterTypeSize();
705 if (I == 0 && hasTemporalProfile()) {
706 uint64_t TimestampValue = swap(*reinterpret_cast<const uint64_t *>(Ptr));
707 if (TimestampValue != 0 &&
708 TimestampValue != std::numeric_limits<uint64_t>::max()) {
709 TemporalProfTimestamps.emplace_back(TimestampValue,
710 swap(Data->NameRef));
711 TemporalProfTraceStreamSize = 1;
712 }
713 if (hasSingleByteCoverage()) {
714 // In coverage mode, getCounterTypeSize() returns 1 byte but our
715 // timestamp field has size uint64_t. Increment I so that the next
716 // iteration of this for loop points to the byte after the timestamp
717 // field, i.e., I += 8.
718 I += 7;
719 }
720 continue;
721 }
722 if (hasSingleByteCoverage()) {
723 // A value of zero signifies the block is covered.
724 Record.Counts.push_back(*Ptr == 0 ? 1 : 0);
725 } else {
726 uint64_t CounterValue = swap(*reinterpret_cast<const uint64_t *>(Ptr));
727 if (CounterValue > MaxCounterValue && Warn)
728 Warn(make_error<InstrProfError>(
730
731 Record.Counts.push_back(CounterValue);
732 }
733 }
734
735 return success();
736}
737
738template <class IntPtrT>
740 uint32_t NumBitmapBytes = swap(Data->NumBitmapBytes);
741
742 Record.BitmapBytes.clear();
743 Record.BitmapBytes.reserve(NumBitmapBytes);
744
745 // It's possible MCDC is either not enabled or only used for some functions
746 // and not others. So if we record 0 bytes, just move on.
747 if (NumBitmapBytes == 0)
748 return success();
749
750 // BitmapDelta decreases as we advance to the next data record.
751 ptrdiff_t BitmapOffset = swap(Data->BitmapPtr) - BitmapDelta;
752 if (BitmapOffset < 0)
753 return error(
755 ("bitmap offset " + Twine(BitmapOffset) + " is negative").str());
756
757 if (BitmapOffset >= BitmapEnd - BitmapStart)
759 ("bitmap offset " + Twine(BitmapOffset) +
760 " is greater than the maximum bitmap offset " +
761 Twine(BitmapEnd - BitmapStart - 1))
762 .str());
763
764 uint64_t MaxNumBitmapBytes =
765 (BitmapEnd - (BitmapStart + BitmapOffset)) / sizeof(uint8_t);
766 if (NumBitmapBytes > MaxNumBitmapBytes)
768 ("number of bitmap bytes " + Twine(NumBitmapBytes) +
769 " is greater than the maximum number of bitmap bytes " +
770 Twine(MaxNumBitmapBytes))
771 .str());
772
773 for (uint32_t I = 0; I < NumBitmapBytes; I++) {
774 const char *Ptr = BitmapStart + BitmapOffset + I;
775 Record.BitmapBytes.push_back(swap(*Ptr));
776 }
777
778 return success();
779}
780
781template <class IntPtrT>
784 Record.clearValueData();
785 CurValueDataSize = 0;
786 // Need to match the logic in value profile dumper code in compiler-rt:
787 uint32_t NumValueKinds = 0;
788 for (uint32_t I = 0; I < IPVK_Last + 1; I++)
789 NumValueKinds += (Data->NumValueSites[I] != 0);
790
791 if (!NumValueKinds)
792 return success();
793
795 ValueProfData::getValueProfData(
796 ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(),
797 getDataEndianness());
798
799 if (Error E = VDataPtrOrErr.takeError())
800 return E;
801
802 // Note that besides deserialization, this also performs the conversion for
803 // indirect call targets. The function pointers from the raw profile are
804 // remapped into function name hashes.
805 VDataPtrOrErr.get()->deserializeTo(Record, Symtab.get());
806 CurValueDataSize = VDataPtrOrErr.get()->getSize();
807 return success();
808}
809
810template <class IntPtrT>
812 // Keep reading profiles that consist of only headers and no profile data and
813 // counters.
814 while (atEnd())
815 // At this point, ValueDataStart field points to the next header.
816 if (Error E = readNextHeader(getNextHeaderPos()))
817 return error(std::move(E));
818
819 // Read name and set it in Record.
820 if (Error E = readName(Record))
821 return error(std::move(E));
822
823 // Read FuncHash and set it in Record.
824 if (Error E = readFuncHash(Record))
825 return error(std::move(E));
826
827 // Read raw counts and set Record.
828 if (Error E = readRawCounts(Record))
829 return error(std::move(E));
830
831 // Read raw bitmap bytes and set Record.
832 if (Error E = readRawBitmapBytes(Record))
833 return error(std::move(E));
834
835 // Read value data and set Record.
836 if (Error E = readValueProfilingData(Record))
837 return error(std::move(E));
838
839 // Iterate.
840 advanceData();
841 return success();
842}
843
844template <class IntPtrT>
846 std::vector<llvm::object::BuildID> &BinaryIds) {
847 BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(),
848 this->BinaryIds.end());
849 return Error::success();
850}
851
852template <class IntPtrT>
854 if (!BinaryIds.empty())
855 printBinaryIdsInternal(OS, BinaryIds);
856 return Error::success();
857}
858
859namespace llvm {
860
861template class RawInstrProfReader<uint32_t>;
862template class RawInstrProfReader<uint64_t>;
863
864} // end namespace llvm
865
868 return IndexedInstrProf::ComputeHash(HashType, K);
869}
870
873
875 const unsigned char *&D, const unsigned char *const End) {
877 ValueProfData::getValueProfData(D, End, ValueProfDataEndianness);
878
879 if (VDataPtrOrErr.takeError())
880 return false;
881
882 VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr);
883 D += VDataPtrOrErr.get()->TotalSize;
884
885 return true;
886}
887
889 offset_type N) {
890 using namespace support;
891
892 // Check if the data is corrupt. If so, don't try to read it.
893 if (N % sizeof(uint64_t))
894 return data_type();
895
896 DataBuffer.clear();
897 std::vector<uint64_t> CounterBuffer;
898 std::vector<uint8_t> BitmapByteBuffer;
899
900 const unsigned char *End = D + N;
901 while (D < End) {
902 // Read hash.
903 if (D + sizeof(uint64_t) >= End)
904 return data_type();
905 uint64_t Hash =
906 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(D);
907
908 // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
909 uint64_t CountsSize = N / sizeof(uint64_t) - 1;
910 // If format version is different then read the number of counters.
911 if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
912 if (D + sizeof(uint64_t) > End)
913 return data_type();
914 CountsSize =
915 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(D);
916 }
917 // Read counter values.
918 if (D + CountsSize * sizeof(uint64_t) > End)
919 return data_type();
920
921 CounterBuffer.clear();
922 CounterBuffer.reserve(CountsSize);
923 for (uint64_t J = 0; J < CountsSize; ++J)
924 CounterBuffer.push_back(
925 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(D));
926
927 // Read bitmap bytes for GET_VERSION(FormatVersion) > 10.
928 if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version10) {
929 uint64_t BitmapBytes = 0;
930 if (D + sizeof(uint64_t) > End)
931 return data_type();
932 BitmapBytes =
933 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(D);
934 // Read bitmap byte values.
935 if (D + BitmapBytes * sizeof(uint8_t) > End)
936 return data_type();
937 BitmapByteBuffer.clear();
938 BitmapByteBuffer.reserve(BitmapBytes);
939 for (uint64_t J = 0; J < BitmapBytes; ++J)
940 BitmapByteBuffer.push_back(static_cast<uint8_t>(
941 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
942 D)));
943 }
944
945 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer),
946 std::move(BitmapByteBuffer));
947
948 // Read value profiling data.
949 if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
951 DataBuffer.clear();
952 return data_type();
953 }
954 }
955 return DataBuffer;
956}
957
958template <typename HashTableImpl>
961 auto Iter = HashTable->find(FuncName);
962 if (Iter == HashTable->end())
963 return make_error<InstrProfError>(instrprof_error::unknown_function);
964
965 Data = (*Iter);
966 if (Data.empty())
967 return make_error<InstrProfError>(instrprof_error::malformed,
968 "profile data is empty");
969
970 return Error::success();
971}
972
973template <typename HashTableImpl>
976 if (atEnd())
977 return make_error<InstrProfError>(instrprof_error::eof);
978
979 Data = *RecordIterator;
980
981 if (Data.empty())
982 return make_error<InstrProfError>(instrprof_error::malformed,
983 "profile data is empty");
984
985 return Error::success();
986}
987
988template <typename HashTableImpl>
990 const unsigned char *Buckets, const unsigned char *const Payload,
991 const unsigned char *const Base, IndexedInstrProf::HashT HashType,
992 uint64_t Version) {
993 FormatVersion = Version;
994 HashTable.reset(HashTableImpl::Create(
995 Buckets, Payload, Base,
996 typename HashTableImpl::InfoType(HashType, Version)));
997 RecordIterator = HashTable->data_begin();
998}
999
1000template <typename HashTableImpl>
1002 return getProfileKindFromVersion(FormatVersion);
1003}
1004
1005namespace {
1006/// A remapper that does not apply any remappings.
1007class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
1008 InstrProfReaderIndexBase &Underlying;
1009
1010public:
1011 InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)
1012 : Underlying(Underlying) {}
1013
1014 Error getRecords(StringRef FuncName,
1015 ArrayRef<NamedInstrProfRecord> &Data) override {
1016 return Underlying.getRecords(FuncName, Data);
1017 }
1018};
1019} // namespace
1020
1021/// A remapper that applies remappings based on a symbol remapping file.
1022template <typename HashTableImpl>
1024 : public InstrProfReaderRemapper {
1025public:
1027 std::unique_ptr<MemoryBuffer> RemapBuffer,
1029 : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
1030 }
1031
1032 /// Extract the original function name from a PGO function name.
1034 // We can have multiple pieces separated by kGlobalIdentifierDelimiter (
1035 // semicolon now and colon in older profiles); there can be pieces both
1036 // before and after the mangled name. Find the first part that starts with
1037 // '_Z'; we'll assume that's the mangled name we want.
1038 std::pair<StringRef, StringRef> Parts = {StringRef(), Name};
1039 while (true) {
1040 Parts = Parts.second.split(kGlobalIdentifierDelimiter);
1041 if (Parts.first.starts_with("_Z"))
1042 return Parts.first;
1043 if (Parts.second.empty())
1044 return Name;
1045 }
1046 }
1047
1048 /// Given a mangled name extracted from a PGO function name, and a new
1049 /// form for that mangled name, reconstitute the name.
1050 static void reconstituteName(StringRef OrigName, StringRef ExtractedName,
1051 StringRef Replacement,
1052 SmallVectorImpl<char> &Out) {
1053 Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());
1054 Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());
1055 Out.insert(Out.end(), Replacement.begin(), Replacement.end());
1056 Out.insert(Out.end(), ExtractedName.end(), OrigName.end());
1057 }
1058
1060 if (Error E = Remappings.read(*RemapBuffer))
1061 return E;
1062 for (StringRef Name : Underlying.HashTable->keys()) {
1063 StringRef RealName = extractName(Name);
1064 if (auto Key = Remappings.insert(RealName)) {
1065 // FIXME: We could theoretically map the same equivalence class to
1066 // multiple names in the profile data. If that happens, we should
1067 // return NamedInstrProfRecords from all of them.
1068 MappedNames.insert({Key, RealName});
1069 }
1070 }
1071 return Error::success();
1072 }
1073
1076 StringRef RealName = extractName(FuncName);
1077 if (auto Key = Remappings.lookup(RealName)) {
1078 StringRef Remapped = MappedNames.lookup(Key);
1079 if (!Remapped.empty()) {
1080 if (RealName.begin() == FuncName.begin() &&
1081 RealName.end() == FuncName.end())
1082 FuncName = Remapped;
1083 else {
1084 // Try rebuilding the name from the given remapping.
1085 SmallString<256> Reconstituted;
1086 reconstituteName(FuncName, RealName, Remapped, Reconstituted);
1087 Error E = Underlying.getRecords(Reconstituted, Data);
1088 if (!E)
1089 return E;
1090
1091 // If we failed because the name doesn't exist, fall back to asking
1092 // about the original name.
1093 if (Error Unhandled = handleErrors(
1094 std::move(E), [](std::unique_ptr<InstrProfError> Err) {
1095 return Err->get() == instrprof_error::unknown_function
1096 ? Error::success()
1097 : Error(std::move(Err));
1098 }))
1099 return Unhandled;
1100 }
1101 }
1102 }
1103 return Underlying.getRecords(FuncName, Data);
1104 }
1105
1106private:
1107 /// The memory buffer containing the remapping configuration. Remappings
1108 /// holds pointers into this buffer.
1109 std::unique_ptr<MemoryBuffer> RemapBuffer;
1110
1111 /// The mangling remapper.
1112 SymbolRemappingReader Remappings;
1113
1114 /// Mapping from mangled name keys to the name used for the key in the
1115 /// profile data.
1116 /// FIXME: Can we store a location within the on-disk hash table instead of
1117 /// redoing lookup?
1119
1120 /// The real profile data reader.
1122};
1123
1125 using namespace support;
1126
1127 if (DataBuffer.getBufferSize() < 8)
1128 return false;
1129 uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
1130 DataBuffer.getBufferStart());
1131 // Verify that it's magical.
1132 return Magic == IndexedInstrProf::Magic;
1133}
1134
1135const unsigned char *
1136IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
1137 const unsigned char *Cur, bool UseCS) {
1138 using namespace IndexedInstrProf;
1139 using namespace support;
1140
1141 if (Version >= IndexedInstrProf::Version4) {
1142 const IndexedInstrProf::Summary *SummaryInLE =
1143 reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
1144 uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
1145 SummaryInLE->NumSummaryFields);
1146 uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
1147 SummaryInLE->NumCutoffEntries);
1148 uint32_t SummarySize =
1149 IndexedInstrProf::Summary::getSize(NFields, NEntries);
1150 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
1151 IndexedInstrProf::allocSummary(SummarySize);
1152
1153 const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
1154 uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
1155 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
1156 Dst[I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[I]);
1157
1158 SummaryEntryVector DetailedSummary;
1159 for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
1160 const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);
1161 DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,
1162 Ent.NumBlocks);
1163 }
1164 std::unique_ptr<llvm::ProfileSummary> &Summary =
1165 UseCS ? this->CS_Summary : this->Summary;
1166
1167 // initialize InstrProfSummary using the SummaryData from disk.
1168 Summary = std::make_unique<ProfileSummary>(
1170 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
1171 SummaryData->get(Summary::MaxBlockCount),
1172 SummaryData->get(Summary::MaxInternalBlockCount),
1173 SummaryData->get(Summary::MaxFunctionCount),
1174 SummaryData->get(Summary::TotalNumBlocks),
1175 SummaryData->get(Summary::TotalNumFunctions));
1176 return Cur + SummarySize;
1177 } else {
1178 // The older versions do not support a profile summary. This just computes
1179 // an empty summary, which will not result in accurate hot/cold detection.
1180 // We would need to call addRecord for all NamedInstrProfRecords to get the
1181 // correct summary. However, this version is old (prior to early 2016) and
1182 // has not been supporting an accurate summary for several years.
1184 Summary = Builder.getSummary();
1185 return Cur;
1186 }
1187}
1188
1190 using namespace support;
1191
1192 const unsigned char *Start =
1193 (const unsigned char *)DataBuffer->getBufferStart();
1194 const unsigned char *Cur = Start;
1195 if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
1197
1198 auto HeaderOr = IndexedInstrProf::Header::readFromBuffer(Start);
1199 if (!HeaderOr)
1200 return HeaderOr.takeError();
1201
1202 const IndexedInstrProf::Header *Header = &HeaderOr.get();
1203 Cur += Header->size();
1204
1205 Cur = readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur,
1206 /* UseCS */ false);
1207 if (Header->formatVersion() & VARIANT_MASK_CSIR_PROF)
1208 Cur =
1209 readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur,
1210 /* UseCS */ true);
1211 // Read the hash type and start offset.
1212 IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>(
1213 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashType));
1214 if (HashType > IndexedInstrProf::HashT::Last)
1216
1217 uint64_t HashOffset =
1218 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashOffset);
1219
1220 // The hash table with profile counts comes next.
1221 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
1222 Start + HashOffset, Cur, Start, HashType, Header->formatVersion());
1223
1224 // The MemProfOffset field in the header is only valid when the format
1225 // version is higher than 8 (when it was introduced).
1226 if (GET_VERSION(Header->formatVersion()) >= 8 &&
1227 Header->formatVersion() & VARIANT_MASK_MEMPROF) {
1228 uint64_t MemProfOffset =
1229 endian::byte_swap<uint64_t, llvm::endianness::little>(
1230 Header->MemProfOffset);
1231
1232 const unsigned char *Ptr = Start + MemProfOffset;
1233 // The value returned from RecordTableGenerator.Emit.
1234 const uint64_t RecordTableOffset =
1236 unaligned>(Ptr);
1237 // The offset in the stream right before invoking
1238 // FrameTableGenerator.Emit.
1239 const uint64_t FramePayloadOffset =
1241 unaligned>(Ptr);
1242 // The value returned from FrameTableGenerator.Emit.
1243 const uint64_t FrameTableOffset =
1245 unaligned>(Ptr);
1246
1247 // Read the schema.
1248 auto SchemaOr = memprof::readMemProfSchema(Ptr);
1249 if (!SchemaOr)
1250 return SchemaOr.takeError();
1251 Schema = SchemaOr.get();
1252
1253 // Now initialize the table reader with a pointer into data buffer.
1254 MemProfRecordTable.reset(MemProfRecordHashTable::Create(
1255 /*Buckets=*/Start + RecordTableOffset,
1256 /*Payload=*/Ptr,
1257 /*Base=*/Start, memprof::RecordLookupTrait(Schema)));
1258
1259 // Initialize the frame table reader with the payload and bucket offsets.
1260 MemProfFrameTable.reset(MemProfFrameHashTable::Create(
1261 /*Buckets=*/Start + FrameTableOffset,
1262 /*Payload=*/Start + FramePayloadOffset,
1263 /*Base=*/Start, memprof::FrameLookupTrait()));
1264
1265#ifdef EXPENSIVE_CHECKS
1266 // Go through all the records and verify that CSId has been correctly
1267 // populated. Do this only under EXPENSIVE_CHECKS. Otherwise, we
1268 // would defeat the purpose of OnDiskIterableChainedHashTable.
1269 for (const auto &Record : MemProfRecordTable->data())
1270 verifyIndexedMemProfRecord(Record);
1271#endif
1272 }
1273
1274 // BinaryIdOffset field in the header is only valid when the format version
1275 // is higher than 9 (when it was introduced).
1276 if (GET_VERSION(Header->formatVersion()) >= 9) {
1277 uint64_t BinaryIdOffset =
1278 endian::byte_swap<uint64_t, llvm::endianness::little>(
1279 Header->BinaryIdOffset);
1280 const unsigned char *Ptr = Start + BinaryIdOffset;
1281 // Read binary ids size.
1282 BinaryIdsSize =
1284 unaligned>(Ptr);
1285 if (BinaryIdsSize % sizeof(uint64_t))
1287 // Set the binary ids start.
1288 BinaryIdsStart = Ptr;
1289 if (BinaryIdsStart > (const unsigned char *)DataBuffer->getBufferEnd())
1290 return make_error<InstrProfError>(instrprof_error::malformed,
1291 "corrupted binary ids");
1292 }
1293
1294 if (GET_VERSION(Header->formatVersion()) >= 12) {
1295 uint64_t VTableNamesOffset =
1296 endian::byte_swap<uint64_t, llvm::endianness::little>(
1297 Header->VTableNamesOffset);
1298 const unsigned char *Ptr = Start + VTableNamesOffset;
1299
1300 CompressedVTableNamesLen =
1302 unaligned>(Ptr);
1303
1304 // Writer first writes the length of compressed string, and then the actual
1305 // content.
1306 VTableNamePtr = (const char *)Ptr;
1307 if (VTableNamePtr > (const char *)DataBuffer->getBufferEnd())
1308 return make_error<InstrProfError>(instrprof_error::truncated);
1309 }
1310
1311 if (GET_VERSION(Header->formatVersion()) >= 10 &&
1312 Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
1313 uint64_t TemporalProfTracesOffset =
1314 endian::byte_swap<uint64_t, llvm::endianness::little>(
1315 Header->TemporalProfTracesOffset);
1316 const unsigned char *Ptr = Start + TemporalProfTracesOffset;
1317 const auto *PtrEnd = (const unsigned char *)DataBuffer->getBufferEnd();
1318 // Expect at least two 64 bit fields: NumTraces, and TraceStreamSize
1319 if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
1321 const uint64_t NumTraces =
1323 unaligned>(Ptr);
1326 unaligned>(Ptr);
1327 for (unsigned i = 0; i < NumTraces; i++) {
1328 // Expect at least two 64 bit fields: Weight and NumFunctions
1329 if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
1332 Trace.Weight =
1334 unaligned>(Ptr);
1335 const uint64_t NumFunctions =
1337 unaligned>(Ptr);
1338 // Expect at least NumFunctions 64 bit fields
1339 if (Ptr + NumFunctions * sizeof(uint64_t) > PtrEnd)
1341 for (unsigned j = 0; j < NumFunctions; j++) {
1342 const uint64_t NameRef =
1344 unaligned>(Ptr);
1345 Trace.FunctionNameRefs.push_back(NameRef);
1346 }
1347 TemporalProfTraces.push_back(std::move(Trace));
1348 }
1349 }
1350
1351 // Load the remapping table now if requested.
1352 if (RemappingBuffer) {
1353 Remapper =
1354 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
1355 std::move(RemappingBuffer), *IndexPtr);
1356 if (Error E = Remapper->populateRemappings())
1357 return E;
1358 } else {
1359 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1360 }
1361 Index = std::move(IndexPtr);
1362
1363 return success();
1364}
1365
1367 if (Symtab)
1368 return *Symtab;
1369
1370 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
1371 if (Error E = Index->populateSymtab(*NewSymtab)) {
1372 auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
1373 consumeError(error(ErrCode, Msg));
1374 }
1375
1376 Symtab = std::move(NewSymtab);
1377 return *Symtab;
1378}
1379
1381 StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName,
1382 uint64_t *MismatchedFuncSum) {
1384 uint64_t FuncSum = 0;
1385 auto Err = Remapper->getRecords(FuncName, Data);
1386 if (Err) {
1387 // If we don't find FuncName, try DeprecatedFuncName to handle profiles
1388 // built by older compilers.
1389 auto Err2 =
1390 handleErrors(std::move(Err), [&](const InstrProfError &IE) -> Error {
1391 if (IE.get() != instrprof_error::unknown_function)
1392 return make_error<InstrProfError>(IE);
1393 if (auto Err = Remapper->getRecords(DeprecatedFuncName, Data))
1394 return Err;
1395 return Error::success();
1396 });
1397 if (Err2)
1398 return std::move(Err2);
1399 }
1400 // Found it. Look for counters with the right hash.
1401
1402 // A flag to indicate if the records are from the same type
1403 // of profile (i.e cs vs nocs).
1404 bool CSBitMatch = false;
1405 auto getFuncSum = [](const std::vector<uint64_t> &Counts) {
1406 uint64_t ValueSum = 0;
1407 for (uint64_t CountValue : Counts) {
1408 if (CountValue == (uint64_t)-1)
1409 continue;
1410 // Handle overflow -- if that happens, return max.
1411 if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1412 return std::numeric_limits<uint64_t>::max();
1413 ValueSum += CountValue;
1414 }
1415 return ValueSum;
1416 };
1417
1418 for (const NamedInstrProfRecord &I : Data) {
1419 // Check for a match and fill the vector if there is one.
1420 if (I.Hash == FuncHash)
1421 return std::move(I);
1424 CSBitMatch = true;
1425 if (MismatchedFuncSum == nullptr)
1426 continue;
1427 FuncSum = std::max(FuncSum, getFuncSum(I.Counts));
1428 }
1429 }
1430 if (CSBitMatch) {
1431 if (MismatchedFuncSum != nullptr)
1432 *MismatchedFuncSum = FuncSum;
1434 }
1436}
1437
1440 // TODO: Add memprof specific errors.
1441 if (MemProfRecordTable == nullptr)
1442 return make_error<InstrProfError>(instrprof_error::invalid_prof,
1443 "no memprof data available in profile");
1444 auto Iter = MemProfRecordTable->find(FuncNameHash);
1445 if (Iter == MemProfRecordTable->end())
1446 return make_error<InstrProfError>(
1448 "memprof record not found for function hash " + Twine(FuncNameHash));
1449
1450 // Setup a callback to convert from frame ids to frame using the on-disk
1451 // FrameData hash table.
1452 memprof::FrameId LastUnmappedFrameId = 0;
1453 bool HasFrameMappingError = false;
1454 auto IdToFrameCallback = [&](const memprof::FrameId Id) {
1455 auto FrIter = MemProfFrameTable->find(Id);
1456 if (FrIter == MemProfFrameTable->end()) {
1457 LastUnmappedFrameId = Id;
1458 HasFrameMappingError = true;
1459 return memprof::Frame(0, 0, 0, false);
1460 }
1461 return *FrIter;
1462 };
1463
1464 memprof::MemProfRecord Record(*Iter, IdToFrameCallback);
1465
1466 // Check that all frame ids were successfully converted to frames.
1467 if (HasFrameMappingError) {
1468 return make_error<InstrProfError>(instrprof_error::hash_mismatch,
1469 "memprof frame not found for frame id " +
1470 Twine(LastUnmappedFrameId));
1471 }
1472 return Record;
1473}
1474
1476 uint64_t FuncHash,
1477 std::vector<uint64_t> &Counts) {
1479 if (Error E = Record.takeError())
1480 return error(std::move(E));
1481
1482 Counts = Record.get().Counts;
1483 return success();
1484}
1485
1487 uint64_t FuncHash,
1488 BitVector &Bitmap) {
1490 if (Error E = Record.takeError())
1491 return error(std::move(E));
1492
1493 const auto &BitmapBytes = Record.get().BitmapBytes;
1494 size_t I = 0, E = BitmapBytes.size();
1495 Bitmap.resize(E * CHAR_BIT);
1497 [&](auto X) {
1498 using XTy = decltype(X);
1499 alignas(XTy) uint8_t W[sizeof(X)];
1500 size_t N = std::min(E - I, sizeof(W));
1501 std::memset(W, 0, sizeof(W));
1502 std::memcpy(W, &BitmapBytes[I], N);
1503 I += N;
1505 support::aligned>(W);
1506 },
1507 Bitmap, Bitmap);
1508 assert(I == E);
1509
1510 return success();
1511}
1512
1515
1516 Error E = Index->getRecords(Data);
1517 if (E)
1518 return error(std::move(E));
1519
1520 Record = Data[RecordIndex++];
1521 if (RecordIndex >= Data.size()) {
1522 Index->advanceToNextKey();
1523 RecordIndex = 0;
1524 }
1525 return success();
1526}
1527
1529 std::vector<llvm::object::BuildID> &BinaryIds) {
1530 return readBinaryIdsInternal(*DataBuffer, BinaryIdsSize, BinaryIdsStart,
1531 BinaryIds, llvm::endianness::little);
1532}
1533
1535 std::vector<llvm::object::BuildID> BinaryIds;
1536 if (Error E = readBinaryIds(BinaryIds))
1537 return E;
1538 printBinaryIdsInternal(OS, BinaryIds);
1539 return Error::success();
1540}
1541
1543 uint64_t NumFuncs = 0;
1544 for (const auto &Func : *this) {
1545 if (isIRLevelProfile()) {
1546 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash);
1547 if (FuncIsCS != IsCS)
1548 continue;
1549 }
1550 Func.accumulateCounts(Sum);
1551 ++NumFuncs;
1552 }
1553 Sum.NumEntries = NumFuncs;
1554}
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file defines the DenseMap class.
std::string Name
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Provides ErrorOr<T> smart pointer.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
static Error initializeReader(InstrProfReader &Reader)
static void printBinaryIdsInternal(raw_ostream &OS, std::vector< llvm::object::BuildID > &BinaryIds)
static Error readBinaryIdsInternal(const MemoryBuffer &DataBuffer, const uint64_t BinaryIdsSize, const uint8_t *BinaryIdsStart, std::vector< llvm::object::BuildID > &BinaryIds, const llvm::endianness Endian)
Read a list of binary ids from a profile that consist of a.
#define READ_NUM(Str, Dst)
#define CHECK_LINE_END(Line)
#define VP_READ_ADVANCE(Val)
static InstrProfKind getProfileKindFromVersion(uint64_t Version)
#define I(x, y, z)
Definition: MD5.cpp:58
if(VerifyEach)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
endianness Endian
raw_pwrite_stream & OS
This file contains some functions that are useful when dealing with strings.
#define error(X)
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
static BitVector & apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args)
Definition: BitVector.h:552
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
Error takeError()
Take ownership of the stored error.
Definition: Error.h:601
reference get()
Returns a reference to the stored T value.
Definition: Error.h:571
Reader for the indexed binary instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< memprof::MemProfRecord > getMemProfRecord(uint64_t FuncNameHash)
Return the memprof record for the function identified by llvm::md5(Name).
Error readHeader() override
Read the file header.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, BitVector &Bitmap)
Fill Bitmap with the profile data for the given function name.
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.
const char * getNamesPointer() const
Return a pointer to the names string that this class constructs.
std::optional< size_t > getDataSize() const
Return the number of ProfileData elements.
size_t getNamesSize() const
Return the number of bytes in the names string.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
Definition: InstrProf.h:394
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
hash_value_type ComputeHash(StringRef K)
ArrayRef< NamedInstrProfRecord > data_type
InstrProfKind getProfileKind() const override
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
A remapper that applies remappings based on a symbol remapping file.
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
Name matcher supporting fuzzy matching of symbol names to names in profiles.
Base class and interface for reading profiling data of any known instrprof format.
std::unique_ptr< InstrProfSymtab > Symtab
Error success()
Clear the current error and return a successful one.
SmallVector< TemporalProfTraceTy > TemporalProfTraces
A list of temporal profile traces.
uint64_t TemporalProfTraceStreamSize
The total number of temporal profile traces seen.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
virtual bool isIRLevelProfile() const =0
virtual Error readHeader()=0
Read the header. Required before reading first record.
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition: InstrProf.h:429
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:540
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Definition: InstrProf.h:518
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:51
size_t getBufferSize() const
Definition: MemoryBuffer.h:68
const char * getBufferEnd() const
Definition: MemoryBuffer.h:67
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
const char * getBufferStart() const
Definition: MemoryBuffer.h:66
static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())
Create the hash table.
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:70
Reader for the raw instrprof binary format from runtime.
Error readHeader() override
Read the header. Required before reading first record.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
static bool hasFormat(const MemoryBuffer &DataBuffer)
InstrProfKind getProfileKind() const override
Returns a BitsetEnum describing the attributes of the raw instr profile.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
SmallVector< TemporalProfTraceTy > & getTemporalProfTraces(std::optional< uint64_t > Weight={}) override
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void reserve(size_type N)
Definition: SmallVector.h:676
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:818
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:696
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:466
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:567
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:257
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
iterator begin() const
Definition: StringRef.h:111
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
iterator end() const
Definition: StringRef.h:113
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition: StringRef.h:729
Reader for symbol remapping files.
Key insert(StringRef FunctionName)
Construct a key for the given symbol, or return an existing one if an equivalent name has already bee...
Key lookup(StringRef FunctionName)
Map the given symbol name into the key for the corresponding equivalence class.
Error read(MemoryBuffer &B)
Read remappings from the given buffer, which must live as long as the remapper.
Reader for the simple text based instrprof format.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error readHeader() override
Read the header.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
LLVM Value Representation.
Definition: Value.h:74
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
Definition: LineIterator.h:63
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
The virtual file system interface.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1163
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1051
const uint64_t Version
Definition: InstrProf.h:1047
const uint64_t Magic
Definition: InstrProf.h:1008
const uint64_t Version
Definition: InstrProf.h:1185
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
Definition: MemProf.cpp:97
value_type read(const void *memory, endianness endian)
Read a value of a particular endianness from memory.
Definition: Endian.h:58
value_type readNext(const CharT *&memory, endianness endian)
Read a value of a particular endianness from a buffer, and increment the buffer past that value.
Definition: Endian.h:76
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
RawInstrProfReader< uint32_t > RawInstrProfReader32
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
Definition: MathExtras.h:382
constexpr T byteswap(T V) noexcept
Reverses the bytes in the given integer value V.
Definition: bit.h:101
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:947
constexpr char kGlobalIdentifierDelimiter
Definition: GlobalValue.h:46
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
std::vector< ProfileSummaryEntry > SummaryEntryVector
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1923
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1858
RawInstrProfReader< uint64_t > RawInstrProfReader64
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:103
endianness
Definition: bit.h:70
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1041
InstrProfKind
An enum describing the attributes of an instrumented profile.
Definition: InstrProf.h:301
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1512
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1087
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1090
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1089
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1123
Profiling information for a single function.
Definition: InstrProf.h:704
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:910
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:355