37#include <system_error>
47 if (Version & VARIANT_MASK_IR_PROF) {
48 ProfileKind |= InstrProfKind::IRInstrumentation;
50 if (Version & VARIANT_MASK_CSIR_PROF) {
51 ProfileKind |= InstrProfKind::ContextSensitive;
53 if (Version & VARIANT_MASK_INSTR_ENTRY) {
54 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
56 if (Version & VARIANT_MASK_BYTE_COVERAGE) {
57 ProfileKind |= InstrProfKind::SingleByteCoverage;
59 if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
60 ProfileKind |= InstrProfKind::FunctionEntryOnly;
62 if (Version & VARIANT_MASK_MEMPROF) {
63 ProfileKind |= InstrProfKind::MemProf;
65 if (Version & VARIANT_MASK_TEMPORAL_PROF) {
66 ProfileKind |= InstrProfKind::TemporalProfile;
74 : FS.getBufferForFile(Filename);
75 if (std::error_code EC = BufferOrErr.getError())
77 return std::move(BufferOrErr.get());
95 const uint8_t *BinaryIdsStart,
96 std::vector<llvm::object::BuildID> &BinaryIds,
98 using namespace support;
100 if (BinaryIdsSize == 0)
103 const uint8_t *BI = BinaryIdsStart;
104 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
106 reinterpret_cast<const uint8_t *
>(DataBuffer.
getBufferEnd());
109 size_t Remaining = BIEnd - BI;
112 return make_error<InstrProfError>(
113 instrprof_error::malformed,
114 "not enough data to read binary id length");
118 BILen = endian::readNext<uint64_t, llvm::endianness::little>(BI);
120 BILen = endian::readNext<uint64_t, llvm::endianness::big>(BI);
123 return make_error<InstrProfError>(instrprof_error::malformed,
124 "binary id length is 0");
126 Remaining = BIEnd - BI;
129 return make_error<InstrProfError>(
130 instrprof_error::malformed,
"not enough data to read binary id data");
138 return make_error<InstrProfError>(
139 instrprof_error::malformed,
140 "binary id section is greater than buffer size");
148 std::vector<llvm::object::BuildID> &BinaryIds) {
149 OS <<
"Binary IDs: \n";
150 for (
auto BI : BinaryIds) {
160 std::function<
void(
Error)> Warn) {
163 if (
Error E = BufferOrError.takeError())
172 std::function<
void(
Error)> Warn) {
173 if (Buffer->getBufferSize() == 0)
176 std::unique_ptr<InstrProfReader> Result;
193 return std::move(Result);
198 const Twine &RemappingPath) {
201 if (
Error E = BufferOrError.takeError())
205 std::unique_ptr<MemoryBuffer> RemappingBuffer;
206 std::string RemappingPathStr = RemappingPath.
str();
207 if (!RemappingPathStr.empty()) {
209 if (
Error E = RemappingBufferOrError.takeError())
211 RemappingBuffer = std::move(RemappingBufferOrError.get());
215 std::move(RemappingBuffer));
220 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
224 auto Result = std::make_unique<IndexedInstrProfReader>(
225 std::move(Buffer), std::move(RemappingBuffer));
231 return std::move(Result);
241 [](
char c) { return isPrint(c) || isSpace(c); });
252 if (Str.equals_insensitive(
"ir"))
254 else if (Str.equals_insensitive(
"fe"))
256 else if (Str.equals_insensitive(
"csir")) {
259 }
else if (Str.equals_insensitive(
"entry_first"))
261 else if (Str.equals_insensitive(
"not_entry_first"))
263 else if (Str.equals_insensitive(
"single_byte_coverage"))
265 else if (Str.equals_insensitive(
"temporal_prof_traces")) {
267 if (
auto Err = readTemporalProfTraceData())
268 return error(std::move(Err));
281Error TextInstrProfReader::readTemporalProfTraceData() {
282 if ((++Line).is_at_end())
289 if ((++Line).is_at_end())
295 for (
uint32_t i = 0; i < NumTraces; i++) {
296 if ((++Line).is_at_end())
303 if ((++Line).is_at_end())
307 Line->
split(FuncNames,
",", -1,
false);
308 for (
auto &FuncName : FuncNames)
309 Trace.FunctionNameRefs.push_back(
319#define CHECK_LINE_END(Line) \
320 if (Line.is_at_end()) \
321 return error(instrprof_error::truncated);
322#define READ_NUM(Str, Dst) \
323 if ((Str).getAsInteger(10, (Dst))) \
324 return error(instrprof_error::malformed);
325#define VP_READ_ADVANCE(Val) \
326 CHECK_LINE_END(Line); \
328 READ_NUM((*Line), (Val)); \
339 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
341 "number of value kinds is invalid");
344 for (
uint32_t VK = 0; VK < NumValueKinds; VK++) {
346 if (ValueKind > IPVK_Last)
353 Record.reserveSites(VK, NumValueSites);
354 for (
uint32_t S = 0; S < NumValueSites; S++) {
357 std::vector<InstrProfValueData> CurrentValues;
360 std::pair<StringRef, StringRef> VD = Line->
rsplit(
':');
362 if (ValueKind == IPVK_IndirectCallTarget) {
370 }
else if (ValueKind == IPVK_VTableTarget) {
382 CurrentValues.push_back({
Value, TakenCount});
385 Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
393#undef VP_READ_ADVANCE
408 return error(std::move(E));
413 if ((Line++)->getAsInteger(0,
Record.Hash))
415 "function hash is not a valid integer");
421 if ((Line++)->getAsInteger(10, NumCounters))
423 "number of counters is not a valid integer");
424 if (NumCounters == 0)
429 Record.Counts.reserve(NumCounters);
434 if ((Line++)->getAsInteger(10, Count))
436 Record.Counts.push_back(Count);
441 Record.BitmapBytes.clear();
444 if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes))
446 "number of bitmap bytes is not a valid integer");
447 if (NumBitmapBytes != 0) {
449 Record.BitmapBytes.reserve(NumBitmapBytes);
450 for (uint8_t
I = 0;
I < NumBitmapBytes; ++
I) {
454 if ((Line++)->getAsInteger(0, BitmapByte))
456 "bitmap byte is not a valid integer");
457 Record.BitmapBytes.push_back(BitmapByte);
464 return error(std::move(E));
469template <
class IntPtrT>
474template <
class IntPtrT>
477 std::optional<uint64_t> Weight) {
478 if (TemporalProfTimestamps.empty()) {
479 assert(TemporalProfTraces.empty());
480 return TemporalProfTraces;
483 std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end());
486 Trace.Weight = *Weight;
487 for (
auto &[TimestampValue, NameRef] : TemporalProfTimestamps)
488 Trace.FunctionNameRefs.push_back(NameRef);
489 TemporalProfTraces = {std::move(
Trace)};
490 return TemporalProfTraces;
493template <
class IntPtrT>
499 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
503template <
class IntPtrT>
505 if (!hasFormat(*DataBuffer))
510 DataBuffer->getBufferStart());
511 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
512 return readHeader(*Header);
515template <
class IntPtrT>
517 const char *
End = DataBuffer->getBufferEnd();
519 while (CurrentPos !=
End && *CurrentPos == 0)
522 if (CurrentPos ==
End)
528 "not enough space for another header");
530 if (
reinterpret_cast<size_t>(CurrentPos) %
alignof(
uint64_t))
532 "insufficient padding");
535 if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
540 return readHeader(*Header);
543template <
class IntPtrT>
546 StringRef(VNamesStart, VNamesEnd - VNamesStart)))
547 return error(std::move(E));
549 const IntPtrT FPtr = swap(
I->FunctionPointer);
555 if (VTableBegin !=
nullptr && VTableEnd !=
nullptr) {
557 I != VTableEnd; ++
I) {
566 swap(
I->VTableNameHash));
572template <
class IntPtrT>
578 (
"Profile uses raw profile format version = " +
579 Twine(GET_VERSION(Version)) +
581 "\nPLEASE update this tool to version in the raw profile, or "
582 "regenerate raw profile with expected version.")
587 const uint8_t *BinaryIdStart =
589 const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;
590 const uint8_t *BufferEnd = (
const uint8_t *)DataBuffer->getBufferEnd();
591 if (BinaryIdSize %
sizeof(
uint64_t) || BinaryIdEnd > BufferEnd)
593 if (BinaryIdSize != 0) {
596 BinaryIds, getDataEndianness()))
600 CountersDelta =
swap(Header.CountersDelta);
601 BitmapDelta =
swap(Header.BitmapDelta);
602 NamesDelta =
swap(Header.NamesDelta);
603 auto NumData =
swap(Header.NumData);
604 auto PaddingBytesBeforeCounters =
swap(Header.PaddingBytesBeforeCounters);
605 auto CountersSize =
swap(Header.NumCounters) * getCounterTypeSize();
606 auto PaddingBytesAfterCounters =
swap(Header.PaddingBytesAfterCounters);
607 auto NumBitmapBytes =
swap(Header.NumBitmapBytes);
608 auto PaddingBytesAfterBitmapBytes =
swap(Header.PaddingBytesAfterBitmapBytes);
609 auto NamesSize =
swap(Header.NamesSize);
610 auto VTableNameSize =
swap(Header.VNamesSize);
611 auto NumVTables =
swap(Header.NumVTables);
612 ValueKindLast =
swap(Header.ValueKindLast);
615 auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
616 auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
618 auto VTableSectionSize =
620 auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
624 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
626 CountersOffset + CountersSize + PaddingBytesAfterCounters;
628 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
630 NamesOffset + NamesSize + PaddingBytesAfterNames;
631 ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
632 PaddingBytesAfterVTableProfData;
634 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
636 auto *Start =
reinterpret_cast<const char *
>(&Header);
637 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
643 if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
646 Data = Correlator->getDataPointer();
653 DataEnd =
Data + NumData;
656 Start + VTableProfDataOffset);
657 VTableEnd = VTableBegin + NumVTables;
658 NamesStart = Start + NamesOffset;
659 NamesEnd = NamesStart + NamesSize;
660 VNamesStart = Start + VTableNameOffset;
661 VNamesEnd = VNamesStart + VTableNameSize;
664 CountersStart = Start + CountersOffset;
665 CountersEnd = CountersStart + CountersSize;
666 BitmapStart = Start + BitmapOffset;
667 BitmapEnd = BitmapStart + NumBitmapBytes;
668 ValueDataStart =
reinterpret_cast<const uint8_t *
>(Start + ValueDataOffset);
670 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
671 if (
Error E = createSymtab(*NewSymtab))
674 Symtab = std::move(NewSymtab);
678template <
class IntPtrT>
684template <
class IntPtrT>
690template <
class IntPtrT>
694 if (NumCounters == 0)
698 if (CounterBaseOffset < 0)
701 (
"counter offset " +
Twine(CounterBaseOffset) +
" is negative").str());
703 if (CounterBaseOffset >= CountersEnd - CountersStart)
705 (
"counter offset " +
Twine(CounterBaseOffset) +
706 " is greater than the maximum counter offset " +
707 Twine(CountersEnd - CountersStart - 1))
711 (CountersEnd - (CountersStart + CounterBaseOffset)) /
712 getCounterTypeSize();
713 if (NumCounters > MaxNumCounters)
715 (
"number of counters " +
Twine(NumCounters) +
716 " is greater than the maximum number of counters " +
717 Twine(MaxNumCounters))
721 Record.Counts.reserve(NumCounters);
724 CountersStart + CounterBaseOffset +
I * getCounterTypeSize();
725 if (
I == 0 && hasTemporalProfile()) {
727 if (TimestampValue != 0 &&
728 TimestampValue != std::numeric_limits<uint64_t>::max()) {
729 TemporalProfTimestamps.emplace_back(TimestampValue,
731 TemporalProfTraceStreamSize = 1;
733 if (hasSingleByteCoverage()) {
742 if (hasSingleByteCoverage()) {
744 Record.Counts.push_back(*
Ptr == 0 ? 1 : 0);
747 if (CounterValue > MaxCounterValue && Warn)
748 Warn(make_error<InstrProfError>(
751 Record.Counts.push_back(CounterValue);
758template <
class IntPtrT>
762 Record.BitmapBytes.clear();
763 Record.BitmapBytes.reserve(NumBitmapBytes);
767 if (NumBitmapBytes == 0)
772 if (BitmapOffset < 0)
775 (
"bitmap offset " +
Twine(BitmapOffset) +
" is negative").str());
777 if (BitmapOffset >= BitmapEnd - BitmapStart)
779 (
"bitmap offset " +
Twine(BitmapOffset) +
780 " is greater than the maximum bitmap offset " +
781 Twine(BitmapEnd - BitmapStart - 1))
785 (BitmapEnd - (BitmapStart + BitmapOffset)) /
sizeof(uint8_t);
786 if (NumBitmapBytes > MaxNumBitmapBytes)
788 (
"number of bitmap bytes " +
Twine(NumBitmapBytes) +
789 " is greater than the maximum number of bitmap bytes " +
790 Twine(MaxNumBitmapBytes))
794 const char *
Ptr = BitmapStart + BitmapOffset +
I;
801template <
class IntPtrT>
805 CurValueDataSize = 0;
809 NumValueKinds += (
Data->NumValueSites[
I] != 0);
815 ValueProfData::getValueProfData(
816 ValueDataStart, (
const unsigned char *)DataBuffer->getBufferEnd(),
817 getDataEndianness());
825 VDataPtrOrErr.
get()->deserializeTo(
Record, Symtab.get());
826 CurValueDataSize = VDataPtrOrErr.
get()->getSize();
830template <
class IntPtrT>
836 if (
Error E = readNextHeader(getNextHeaderPos()))
837 return error(std::move(E));
841 return error(std::move(E));
845 return error(std::move(E));
849 return error(std::move(E));
853 return error(std::move(E));
857 return error(std::move(E));
864template <
class IntPtrT>
866 std::vector<llvm::object::BuildID> &BinaryIds) {
867 BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(),
868 this->BinaryIds.end());
872template <
class IntPtrT>
874 if (!BinaryIds.empty())
895 const unsigned char *&
D,
const unsigned char *
const End) {
897 ValueProfData::getValueProfData(
D,
End, ValueProfDataEndianness);
902 VDataPtrOrErr.
get()->deserializeTo(DataBuffer.back(),
nullptr);
903 D += VDataPtrOrErr.
get()->TotalSize;
910 using namespace support;
917 std::vector<uint64_t> CounterBuffer;
918 std::vector<uint8_t> BitmapByteBuffer;
920 const unsigned char *
End =
D +
N;
925 uint64_t Hash = endian::readNext<uint64_t, llvm::endianness::little>(
D);
933 CountsSize = endian::readNext<uint64_t, llvm::endianness::little>(
D);
939 CounterBuffer.clear();
940 CounterBuffer.reserve(CountsSize);
941 for (
uint64_t J = 0; J < CountsSize; ++J)
942 CounterBuffer.push_back(
943 endian::readNext<uint64_t, llvm::endianness::little>(
D));
950 BitmapBytes = endian::readNext<uint64_t, llvm::endianness::little>(
D);
952 if (
D + BitmapBytes *
sizeof(uint8_t) >
End)
954 BitmapByteBuffer.clear();
955 BitmapByteBuffer.reserve(BitmapBytes);
956 for (
uint64_t J = 0; J < BitmapBytes; ++J)
957 BitmapByteBuffer.push_back(
static_cast<uint8_t
>(
958 endian::readNext<uint64_t, llvm::endianness::little>(
D)));
961 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer),
962 std::move(BitmapByteBuffer));
974template <
typename HashTableImpl>
977 auto Iter = HashTable->find(FuncName);
978 if (Iter == HashTable->end())
984 "profile data is empty");
989template <
typename HashTableImpl>
995 Data = *RecordIterator;
999 "profile data is empty");
1004template <
typename HashTableImpl>
1006 const unsigned char *Buckets,
const unsigned char *
const Payload,
1009 FormatVersion = Version;
1010 HashTable.reset(HashTableImpl::Create(
1011 Buckets, Payload,
Base,
1012 typename HashTableImpl::InfoType(HashType, Version)));
1013 RecordIterator = HashTable->data_begin();
1016template <
typename HashTableImpl>
1028 : Underlying(Underlying) {}
1032 return Underlying.getRecords(FuncName, Data);
1038template <
typename HashTableImpl>
1043 std::unique_ptr<MemoryBuffer> RemapBuffer,
1045 : RemapBuffer(
std::
move(RemapBuffer)), Underlying(Underlying) {
1054 std::pair<StringRef, StringRef> Parts = {
StringRef(),
Name};
1057 if (Parts.first.starts_with(
"_Z"))
1059 if (Parts.second.empty())
1076 if (
Error E = Remappings.
read(*RemapBuffer))
1080 if (
auto Key = Remappings.
insert(RealName)) {
1084 MappedNames.insert({Key, RealName});
1093 if (
auto Key = Remappings.
lookup(RealName)) {
1094 StringRef Remapped = MappedNames.lookup(Key);
1095 if (!Remapped.
empty()) {
1097 RealName.
end() == FuncName.
end())
1098 FuncName = Remapped;
1103 Error E = Underlying.getRecords(Reconstituted,
Data);
1110 std::move(E), [](std::unique_ptr<InstrProfError> Err) {
1113 :
Error(std::move(Err));
1119 return Underlying.getRecords(FuncName,
Data);
1125 std::unique_ptr<MemoryBuffer> RemapBuffer;
1141 using namespace support;
1145 uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
1151const unsigned char *
1153 const unsigned char *Cur,
bool UseCS) {
1154 using namespace IndexedInstrProf;
1155 using namespace support;
1160 uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
1162 uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
1166 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
1171 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
1172 Dst[
I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[
I]);
1175 for (
unsigned I = 0;
I < SummaryData->NumCutoffEntries;
I++) {
1180 std::unique_ptr<llvm::ProfileSummary> &Summary =
1181 UseCS ? this->CS_Summary : this->Summary;
1184 Summary = std::make_unique<ProfileSummary>(
1186 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
1187 SummaryData->get(Summary::MaxBlockCount),
1188 SummaryData->get(Summary::MaxInternalBlockCount),
1189 SummaryData->get(Summary::MaxFunctionCount),
1190 SummaryData->get(Summary::TotalNumBlocks),
1191 SummaryData->get(Summary::TotalNumFunctions));
1192 return Cur + SummarySize;
1200 Summary = Builder.getSummary();
1207 const unsigned char *
Ptr = Start + MemProfOffset;
1213 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1219 }
else if (FirstWord >= 24) {
1226 return make_error<InstrProfError>(
1228 formatv(
"MemProf version {} not supported; "
1229 "requires version between {} and {}, inclusive",
1238 : support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1241 const uint64_t FramePayloadOffset =
1242 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1245 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1249 uint64_t CallStackPayloadOffset = 0;
1253 CallStackPayloadOffset =
1254 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1255 CallStackTableOffset =
1256 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1262 return SchemaOr.takeError();
1263 Schema = SchemaOr.get();
1267 Start + RecordTableOffset,
1273 Start + FrameTableOffset,
1274 Start + FramePayloadOffset,
1279 Start + CallStackTableOffset,
1280 Start + CallStackPayloadOffset,
1283#ifdef EXPENSIVE_CHECKS
1291 for (
const auto &
Record : MemProfRecordTable->data())
1292 verifyIndexedMemProfRecord(
Record);
1299 using namespace support;
1301 const unsigned char *Start =
1302 (
const unsigned char *)DataBuffer->getBufferStart();
1303 const unsigned char *Cur = Start;
1304 if ((
const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
1309 return HeaderOr.takeError();
1312 Cur += Header->size();
1316 if (Header->formatVersion() & VARIANT_MASK_CSIR_PROF)
1322 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashType));
1327 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashOffset);
1330 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
1331 Start + HashOffset, Cur, Start, HashType, Header->formatVersion());
1335 if (GET_VERSION(Header->formatVersion()) >= 8 &&
1336 Header->formatVersion() & VARIANT_MASK_MEMPROF) {
1338 endian::byte_swap<uint64_t, llvm::endianness::little>(
1339 Header->MemProfOffset);
1346 if (GET_VERSION(Header->formatVersion()) >= 9) {
1348 endian::byte_swap<uint64_t, llvm::endianness::little>(
1349 Header->BinaryIdOffset);
1350 const unsigned char *
Ptr = Start + BinaryIdOffset;
1353 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1354 if (BinaryIdsSize %
sizeof(
uint64_t))
1357 BinaryIdsStart =
Ptr;
1358 if (BinaryIdsStart > (
const unsigned char *)DataBuffer->getBufferEnd())
1360 "corrupted binary ids");
1363 if (GET_VERSION(Header->formatVersion()) >= 12) {
1365 endian::byte_swap<uint64_t, llvm::endianness::little>(
1366 Header->VTableNamesOffset);
1367 const unsigned char *
Ptr = Start + VTableNamesOffset;
1369 CompressedVTableNamesLen =
1370 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1374 VTableNamePtr = (
const char *)
Ptr;
1375 if (VTableNamePtr > (
const char *)DataBuffer->getBufferEnd())
1379 if (GET_VERSION(Header->formatVersion()) >= 10 &&
1380 Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
1381 uint64_t TemporalProfTracesOffset =
1382 endian::byte_swap<uint64_t, llvm::endianness::little>(
1383 Header->TemporalProfTracesOffset);
1384 const unsigned char *
Ptr = Start + TemporalProfTracesOffset;
1385 const auto *PtrEnd = (
const unsigned char *)DataBuffer->getBufferEnd();
1390 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1392 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1393 for (
unsigned i = 0; i < NumTraces; i++) {
1399 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1401 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1403 if (
Ptr + NumFunctions *
sizeof(
uint64_t) > PtrEnd)
1405 for (
unsigned j = 0; j < NumFunctions; j++) {
1407 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1408 Trace.FunctionNameRefs.push_back(NameRef);
1415 if (RemappingBuffer) {
1417 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
1418 std::move(RemappingBuffer), *IndexPtr);
1419 if (
Error E = Remapper->populateRemappings())
1422 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1424 Index = std::move(IndexPtr);
1433 auto NewSymtab = std::make_unique<InstrProfSymtab>();
1435 if (
Error E = NewSymtab->initVTableNamesFromCompressedStrings(
1436 StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
1442 if (
Error E =
Index->populateSymtab(*NewSymtab)) {
1447 Symtab = std::move(NewSymtab);
1456 auto Err = Remapper->getRecords(FuncName,
Data);
1463 return make_error<InstrProfError>(IE);
1464 if (
auto Err = Remapper->getRecords(DeprecatedFuncName,
Data))
1469 return std::move(Err2);
1475 bool CSBitMatch =
false;
1476 auto getFuncSum = [](
const std::vector<uint64_t> &Counts) {
1478 for (
uint64_t CountValue : Counts) {
1482 if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1483 return std::numeric_limits<uint64_t>::max();
1484 ValueSum += CountValue;
1491 if (
I.Hash == FuncHash)
1492 return std::move(
I);
1496 if (MismatchedFuncSum ==
nullptr)
1498 FuncSum = std::max(FuncSum, getFuncSum(
I.Counts));
1502 if (MismatchedFuncSum !=
nullptr)
1503 *MismatchedFuncSum = FuncSum;
1512 if (MemProfRecordTable ==
nullptr)
1514 "no memprof data available in profile");
1515 auto Iter = MemProfRecordTable->find(FuncNameHash);
1516 if (Iter == MemProfRecordTable->end())
1517 return make_error<InstrProfError>(
1519 "memprof record not found for function hash " +
Twine(FuncNameHash));
1523 std::optional<memprof::FrameId> LastUnmappedFrameId;
1525 auto FrIter = MemProfFrameTable->find(Id);
1526 if (FrIter == MemProfFrameTable->end()) {
1527 LastUnmappedFrameId = Id;
1535 std::optional<memprof::CallStackId> LastUnmappedCSId;
1538 auto CSIter = MemProfCallStackTable->find(CSId);
1539 if (CSIter == MemProfCallStackTable->end()) {
1540 LastUnmappedCSId = CSId;
1545 Frames.
push_back(IdToFrameCallback(Id));
1552 if (MemProfCallStackTable)
1558 if (LastUnmappedFrameId) {
1560 "memprof frame not found for frame id " +
1561 Twine(*LastUnmappedFrameId));
1565 if (LastUnmappedCSId) {
1566 return make_error<InstrProfError>(
1568 "memprof call stack not found for call stack id " +
1569 Twine(*LastUnmappedCSId));
1576 std::vector<uint64_t> &Counts) {
1579 return error(std::move(E));
1581 Counts =
Record.get().Counts;
1590 return error(std::move(E));
1592 const auto &BitmapBytes =
Record.get().BitmapBytes;
1593 size_t I = 0, E = BitmapBytes.size();
1594 Bitmap.
resize(E * CHAR_BIT);
1597 using XTy =
decltype(
X);
1598 alignas(XTy) uint8_t W[
sizeof(
X)];
1599 size_t N = std::min(E -
I,
sizeof(W));
1600 std::memset(W, 0,
sizeof(W));
1601 std::memcpy(W, &BitmapBytes[
I],
N);
1617 return error(std::move(E));
1620 if (RecordIndex >=
Data.size()) {
1621 Index->advanceToNextKey();
1628 std::vector<llvm::object::BuildID> &BinaryIds) {
1634 std::vector<llvm::object::BuildID> BinaryIds;
1643 for (
const auto &Func : *
this) {
1646 if (FuncIsCS != IsCS)
1649 Func.accumulateCounts(Sum);
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Provides ErrorOr<T> smart pointer.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
static Error initializeReader(InstrProfReader &Reader)
static void printBinaryIdsInternal(raw_ostream &OS, std::vector< llvm::object::BuildID > &BinaryIds)
static Error readBinaryIdsInternal(const MemoryBuffer &DataBuffer, const uint64_t BinaryIdsSize, const uint8_t *BinaryIdsStart, std::vector< llvm::object::BuildID > &BinaryIds, const llvm::endianness Endian)
Read a list of binary ids from a profile that consist of a.
#define READ_NUM(Str, Dst)
#define CHECK_LINE_END(Line)
#define VP_READ_ADVANCE(Val)
static InstrProfKind getProfileKindFromVersion(uint64_t Version)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
static BitVector & apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args)
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
reference get()
Returns a reference to the stored T value.
Reader for the indexed binary instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Error readHeader() override
Read the file header.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, BitVector &Bitmap)
Fill Bitmap with the profile data for the given function name.
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
Error deserialize(const unsigned char *Start, uint64_t MemProfOffset)
Expected< memprof::MemProfRecord > getMemProfRecord(const uint64_t FuncNameHash) const
InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.
const char * getNamesPointer() const
Return a pointer to the names string that this class constructs.
std::optional< size_t > getDataSize() const
Return the number of ProfileData elements.
size_t getNamesSize() const
Return the number of bytes in the names string.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
hash_value_type ComputeHash(StringRef K)
ArrayRef< NamedInstrProfRecord > data_type
InstrProfKind getProfileKind() const override
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
A remapper that applies remappings based on a symbol remapping file.
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
Error populateRemappings() override
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
Name matcher supporting fuzzy matching of symbol names to names in profiles.
Base class and interface for reading profiling data of any known instrprof format.
std::unique_ptr< InstrProfSymtab > Symtab
Error success()
Clear the current error and return a successful one.
SmallVector< TemporalProfTraceTy > TemporalProfTraces
A list of temporal profile traces.
uint64_t TemporalProfTraceStreamSize
The total number of temporal profile traces seen.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
virtual bool isIRLevelProfile() const =0
virtual Error readHeader()=0
Read the header. Required before reading first record.
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val)
Map the address range (i.e., [start_address, end_address)) of a variable to its names' MD5 hash.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
size_t getBufferSize() const
const char * getBufferEnd() const
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
const char * getBufferStart() const
static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())
Create the hash table.
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Reader for the raw instrprof binary format from runtime.
Error readHeader() override
Read the header. Required before reading first record.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
static bool hasFormat(const MemoryBuffer &DataBuffer)
InstrProfKind getProfileKind() const override
Returns a BitsetEnum describing the attributes of the raw instr profile.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
SmallVector< TemporalProfTraceTy > & getTemporalProfTraces(std::optional< uint64_t > Weight={}) override
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr size_t size() const
size - Get the string size.
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Reader for symbol remapping files.
Key insert(StringRef FunctionName)
Construct a key for the given symbol, or return an existing one if an equivalent name has already bee...
Key lookup(StringRef FunctionName)
Map the given symbol name into the key for the corresponding equivalence class.
Error read(MemoryBuffer &B)
Read remappings from the given buffer, which must live as long as the remapper.
Reader for the simple text based instrprof format.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error readHeader() override
Read the header.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
LLVM Value Representation.
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
This class implements an extremely fast bulk output stream that can only output to a stream.
The virtual file system interface.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
constexpr uint64_t MaximumSupportedVersion
constexpr uint64_t MinimumSupportedVersion
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
value_type read(const void *memory, endianness endian)
Read a value of a particular endianness from memory.
This is an optimization pass for GlobalISel generic memory operations.
RawInstrProfReader< uint32_t > RawInstrProfReader32
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
constexpr T byteswap(T V) noexcept
Reverses the bytes in the given integer value V.
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
constexpr char kGlobalIdentifierDelimiter
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
std::vector< ProfileSummaryEntry > SummaryEntryVector
@ raw_profile_version_mismatch
@ counter_value_too_large
@ unexpected_correlation_info
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
RawInstrProfReader< uint64_t > RawInstrProfReader64
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
void consumeError(Error Err)
Consume a Error without doing anything.
InstrProfKind
An enum describing the attributes of an instrumented profile.
@ FunctionEntryInstrumentation
@ FrontendInstrumentation
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
uint64_t Cutoff
The required percentile of total execution count.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumSummaryFields
uint64_t NumCutoffEntries
Profiling information for a single function.
static bool hasCSFlagInHash(uint64_t FuncHash)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
MemProfRecord toMemProfRecord(std::function< const llvm::SmallVector< Frame >(const CallStackId)> Callback) const