37#include <system_error>
47 if (Version & VARIANT_MASK_IR_PROF) {
48 ProfileKind |= InstrProfKind::IRInstrumentation;
50 if (Version & VARIANT_MASK_CSIR_PROF) {
51 ProfileKind |= InstrProfKind::ContextSensitive;
53 if (Version & VARIANT_MASK_INSTR_ENTRY) {
54 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
56 if (Version & VARIANT_MASK_BYTE_COVERAGE) {
57 ProfileKind |= InstrProfKind::SingleByteCoverage;
59 if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
60 ProfileKind |= InstrProfKind::FunctionEntryOnly;
62 if (Version & VARIANT_MASK_MEMPROF) {
63 ProfileKind |= InstrProfKind::MemProf;
65 if (Version & VARIANT_MASK_TEMPORAL_PROF) {
66 ProfileKind |= InstrProfKind::TemporalProfile;
74 : FS.getBufferForFile(Filename);
75 if (std::error_code EC = BufferOrErr.getError())
77 return std::move(BufferOrErr.get());
95 std::vector<llvm::object::BuildID> &BinaryIds,
97 using namespace support;
100 const uint8_t *BinaryIdsStart = BinaryIdsBuffer.
data();
102 if (BinaryIdsSize == 0)
105 const uint8_t *BI = BinaryIdsStart;
106 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
108 reinterpret_cast<const uint8_t *
>(DataBuffer.
getBufferEnd());
111 size_t Remaining = BIEnd - BI;
114 return make_error<InstrProfError>(
115 instrprof_error::malformed,
116 "not enough data to read binary id length");
120 return make_error<InstrProfError>(instrprof_error::malformed,
121 "binary id length is 0");
123 Remaining = BIEnd - BI;
126 return make_error<InstrProfError>(
127 instrprof_error::malformed,
"not enough data to read binary id data");
135 return make_error<InstrProfError>(
136 instrprof_error::malformed,
137 "binary id section is greater than buffer size");
145 OS <<
"Binary IDs: \n";
146 for (
const auto &BI : BinaryIds) {
156 std::function<
void(
Error)> Warn) {
159 if (
Error E = BufferOrError.takeError())
168 std::function<
void(
Error)> Warn) {
169 if (Buffer->getBufferSize() == 0)
172 std::unique_ptr<InstrProfReader> Result;
189 return std::move(Result);
194 const Twine &RemappingPath) {
197 if (
Error E = BufferOrError.takeError())
201 std::unique_ptr<MemoryBuffer> RemappingBuffer;
202 std::string RemappingPathStr = RemappingPath.
str();
203 if (!RemappingPathStr.empty()) {
205 if (
Error E = RemappingBufferOrError.takeError())
207 RemappingBuffer = std::move(RemappingBufferOrError.get());
211 std::move(RemappingBuffer));
216 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
220 auto Result = std::make_unique<IndexedInstrProfReader>(
221 std::move(Buffer), std::move(RemappingBuffer));
227 return std::move(Result);
237 [](
char c) { return isPrint(c) || isSpace(c); });
248 if (Str.equals_insensitive(
"ir"))
250 else if (Str.equals_insensitive(
"fe"))
252 else if (Str.equals_insensitive(
"csir")) {
255 }
else if (Str.equals_insensitive(
"entry_first"))
257 else if (Str.equals_insensitive(
"not_entry_first"))
259 else if (Str.equals_insensitive(
"single_byte_coverage"))
261 else if (Str.equals_insensitive(
"temporal_prof_traces")) {
263 if (
auto Err = readTemporalProfTraceData())
264 return error(std::move(Err));
277Error TextInstrProfReader::readTemporalProfTraceData() {
278 if ((++Line).is_at_end())
285 if ((++Line).is_at_end())
291 for (
uint32_t i = 0; i < NumTraces; i++) {
292 if ((++Line).is_at_end())
299 if ((++Line).is_at_end())
303 Line->
split(FuncNames,
",", -1,
false);
304 for (
auto &FuncName : FuncNames)
305 Trace.FunctionNameRefs.push_back(
315#define CHECK_LINE_END(Line) \
316 if (Line.is_at_end()) \
317 return error(instrprof_error::truncated);
318#define READ_NUM(Str, Dst) \
319 if ((Str).getAsInteger(10, (Dst))) \
320 return error(instrprof_error::malformed);
321#define VP_READ_ADVANCE(Val) \
322 CHECK_LINE_END(Line); \
324 READ_NUM((*Line), (Val)); \
335 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
337 "number of value kinds is invalid");
340 for (
uint32_t VK = 0; VK < NumValueKinds; VK++) {
342 if (ValueKind > IPVK_Last)
349 Record.reserveSites(VK, NumValueSites);
350 for (
uint32_t S = 0; S < NumValueSites; S++) {
353 std::vector<InstrProfValueData> CurrentValues;
356 std::pair<StringRef, StringRef> VD = Line->
rsplit(
':');
358 if (ValueKind == IPVK_IndirectCallTarget) {
366 }
else if (ValueKind == IPVK_VTableTarget) {
378 CurrentValues.push_back({
Value, TakenCount});
381 assert(CurrentValues.size() == NumValueData);
382 Record.addValueData(ValueKind, S, CurrentValues,
nullptr);
389#undef VP_READ_ADVANCE
404 return error(std::move(E));
409 if ((Line++)->getAsInteger(0,
Record.Hash))
411 "function hash is not a valid integer");
417 if ((Line++)->getAsInteger(10, NumCounters))
419 "number of counters is not a valid integer");
420 if (NumCounters == 0)
425 Record.Counts.reserve(NumCounters);
430 if ((Line++)->getAsInteger(10, Count))
432 Record.Counts.push_back(Count);
437 Record.BitmapBytes.clear();
440 if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes))
442 "number of bitmap bytes is not a valid integer");
443 if (NumBitmapBytes != 0) {
445 Record.BitmapBytes.reserve(NumBitmapBytes);
446 for (uint8_t
I = 0;
I < NumBitmapBytes; ++
I) {
450 if ((Line++)->getAsInteger(0, BitmapByte))
452 "bitmap byte is not a valid integer");
453 Record.BitmapBytes.push_back(BitmapByte);
460 return error(std::move(E));
465template <
class IntPtrT>
470template <
class IntPtrT>
473 std::optional<uint64_t> Weight) {
474 if (TemporalProfTimestamps.empty()) {
475 assert(TemporalProfTraces.empty());
476 return TemporalProfTraces;
479 std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end());
482 Trace.Weight = *Weight;
483 for (
auto &[TimestampValue, NameRef] : TemporalProfTimestamps)
484 Trace.FunctionNameRefs.push_back(NameRef);
485 TemporalProfTraces = {std::move(
Trace)};
486 return TemporalProfTraces;
489template <
class IntPtrT>
495 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
499template <
class IntPtrT>
501 if (!hasFormat(*DataBuffer))
506 DataBuffer->getBufferStart());
507 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
508 return readHeader(*Header);
511template <
class IntPtrT>
513 const char *
End = DataBuffer->getBufferEnd();
515 while (CurrentPos !=
End && *CurrentPos == 0)
518 if (CurrentPos ==
End)
524 "not enough space for another header");
526 if (
reinterpret_cast<size_t>(CurrentPos) %
alignof(
uint64_t))
528 "insufficient padding");
531 if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
536 return readHeader(*Header);
539template <
class IntPtrT>
542 StringRef(VNamesStart, VNamesEnd - VNamesStart)))
543 return error(std::move(E));
545 const IntPtrT FPtr = swap(
I->FunctionPointer);
551 if (VTableBegin !=
nullptr && VTableEnd !=
nullptr) {
553 I != VTableEnd; ++
I) {
562 swap(
I->VTableNameHash));
568template <
class IntPtrT>
574 (
"Profile uses raw profile format version = " +
577 "\nPLEASE update this tool to version in the raw profile, or "
578 "regenerate raw profile with expected version.")
583 const uint8_t *BinaryIdStart =
585 const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;
586 const uint8_t *BufferEnd = (
const uint8_t *)DataBuffer->getBufferEnd();
587 if (BinaryIdSize %
sizeof(
uint64_t) || BinaryIdEnd > BufferEnd)
590 if (!BinaryIdsBuffer.empty()) {
592 BinaryIds, getDataEndianness()))
596 CountersDelta =
swap(Header.CountersDelta);
597 BitmapDelta =
swap(Header.BitmapDelta);
598 NamesDelta =
swap(Header.NamesDelta);
599 auto NumData =
swap(Header.NumData);
600 auto PaddingBytesBeforeCounters =
swap(Header.PaddingBytesBeforeCounters);
601 auto CountersSize =
swap(Header.NumCounters) * getCounterTypeSize();
602 auto PaddingBytesAfterCounters =
swap(Header.PaddingBytesAfterCounters);
603 auto NumBitmapBytes =
swap(Header.NumBitmapBytes);
604 auto PaddingBytesAfterBitmapBytes =
swap(Header.PaddingBytesAfterBitmapBytes);
605 auto NamesSize =
swap(Header.NamesSize);
606 auto VTableNameSize =
swap(Header.VNamesSize);
607 auto NumVTables =
swap(Header.NumVTables);
608 ValueKindLast =
swap(Header.ValueKindLast);
611 auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
612 auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
614 auto VTableSectionSize =
616 auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
620 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
622 CountersOffset + CountersSize + PaddingBytesAfterCounters;
624 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
626 NamesOffset + NamesSize + PaddingBytesAfterNames;
627 ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
628 PaddingBytesAfterVTableProfData;
630 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
632 auto *Start =
reinterpret_cast<const char *
>(&Header);
633 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
639 if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
642 Data = Correlator->getDataPointer();
649 DataEnd =
Data + NumData;
652 Start + VTableProfDataOffset);
653 VTableEnd = VTableBegin + NumVTables;
654 NamesStart = Start + NamesOffset;
655 NamesEnd = NamesStart + NamesSize;
656 VNamesStart = Start + VTableNameOffset;
657 VNamesEnd = VNamesStart + VTableNameSize;
660 CountersStart = Start + CountersOffset;
661 CountersEnd = CountersStart + CountersSize;
662 BitmapStart = Start + BitmapOffset;
663 BitmapEnd = BitmapStart + NumBitmapBytes;
664 ValueDataStart =
reinterpret_cast<const uint8_t *
>(Start + ValueDataOffset);
666 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
667 if (
Error E = createSymtab(*NewSymtab))
670 Symtab = std::move(NewSymtab);
674template <
class IntPtrT>
680template <
class IntPtrT>
686template <
class IntPtrT>
690 if (NumCounters == 0)
694 if (CounterBaseOffset < 0)
697 (
"counter offset " +
Twine(CounterBaseOffset) +
" is negative").str());
699 if (CounterBaseOffset >= CountersEnd - CountersStart)
701 (
"counter offset " +
Twine(CounterBaseOffset) +
702 " is greater than the maximum counter offset " +
703 Twine(CountersEnd - CountersStart - 1))
707 (CountersEnd - (CountersStart + CounterBaseOffset)) /
708 getCounterTypeSize();
709 if (NumCounters > MaxNumCounters)
711 (
"number of counters " +
Twine(NumCounters) +
712 " is greater than the maximum number of counters " +
713 Twine(MaxNumCounters))
717 Record.Counts.reserve(NumCounters);
720 CountersStart + CounterBaseOffset +
I * getCounterTypeSize();
721 if (
I == 0 && hasTemporalProfile()) {
723 if (TimestampValue != 0 &&
724 TimestampValue != std::numeric_limits<uint64_t>::max()) {
725 TemporalProfTimestamps.emplace_back(TimestampValue,
727 TemporalProfTraceStreamSize = 1;
729 if (hasSingleByteCoverage()) {
738 if (hasSingleByteCoverage()) {
740 Record.Counts.push_back(*
Ptr == 0 ? 1 : 0);
743 if (CounterValue > MaxCounterValue && Warn)
744 Warn(make_error<InstrProfError>(
747 Record.Counts.push_back(CounterValue);
754template <
class IntPtrT>
758 Record.BitmapBytes.clear();
759 Record.BitmapBytes.reserve(NumBitmapBytes);
763 if (NumBitmapBytes == 0)
768 if (BitmapOffset < 0)
771 (
"bitmap offset " +
Twine(BitmapOffset) +
" is negative").str());
773 if (BitmapOffset >= BitmapEnd - BitmapStart)
775 (
"bitmap offset " +
Twine(BitmapOffset) +
776 " is greater than the maximum bitmap offset " +
777 Twine(BitmapEnd - BitmapStart - 1))
781 (BitmapEnd - (BitmapStart + BitmapOffset)) /
sizeof(uint8_t);
782 if (NumBitmapBytes > MaxNumBitmapBytes)
784 (
"number of bitmap bytes " +
Twine(NumBitmapBytes) +
785 " is greater than the maximum number of bitmap bytes " +
786 Twine(MaxNumBitmapBytes))
790 const char *
Ptr = BitmapStart + BitmapOffset +
I;
797template <
class IntPtrT>
801 CurValueDataSize = 0;
805 NumValueKinds += (
Data->NumValueSites[
I] != 0);
811 ValueProfData::getValueProfData(
812 ValueDataStart, (
const unsigned char *)DataBuffer->getBufferEnd(),
813 getDataEndianness());
821 VDataPtrOrErr.
get()->deserializeTo(
Record, Symtab.get());
822 CurValueDataSize = VDataPtrOrErr.
get()->getSize();
826template <
class IntPtrT>
832 if (
Error E = readNextHeader(getNextHeaderPos()))
833 return error(std::move(E));
837 return error(std::move(E));
841 return error(std::move(E));
845 return error(std::move(E));
849 return error(std::move(E));
853 return error(std::move(E));
860template <
class IntPtrT>
862 std::vector<llvm::object::BuildID> &BinaryIds) {
863 BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(),
864 this->BinaryIds.end());
868template <
class IntPtrT>
870 if (!BinaryIds.empty())
891 const unsigned char *&
D,
const unsigned char *
const End) {
893 ValueProfData::getValueProfData(
D,
End, ValueProfDataEndianness);
898 VDataPtrOrErr.
get()->deserializeTo(DataBuffer.back(),
nullptr);
899 D += VDataPtrOrErr.
get()->TotalSize;
906 using namespace support;
913 std::vector<uint64_t> CounterBuffer;
914 std::vector<uint8_t> BitmapByteBuffer;
916 const unsigned char *
End =
D +
N;
921 uint64_t Hash = endian::readNext<uint64_t, llvm::endianness::little>(
D);
929 CountsSize = endian::readNext<uint64_t, llvm::endianness::little>(
D);
935 CounterBuffer.clear();
936 CounterBuffer.reserve(CountsSize);
937 for (
uint64_t J = 0; J < CountsSize; ++J)
938 CounterBuffer.push_back(
939 endian::readNext<uint64_t, llvm::endianness::little>(
D));
946 BitmapBytes = endian::readNext<uint64_t, llvm::endianness::little>(
D);
948 if (
D + BitmapBytes *
sizeof(uint8_t) >
End)
950 BitmapByteBuffer.clear();
951 BitmapByteBuffer.reserve(BitmapBytes);
952 for (
uint64_t J = 0; J < BitmapBytes; ++J)
953 BitmapByteBuffer.push_back(
static_cast<uint8_t
>(
954 endian::readNext<uint64_t, llvm::endianness::little>(
D)));
957 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer),
958 std::move(BitmapByteBuffer));
970template <
typename HashTableImpl>
973 auto Iter = HashTable->find(FuncName);
974 if (Iter == HashTable->end())
980 "profile data is empty");
985template <
typename HashTableImpl>
991 Data = *RecordIterator;
995 "profile data is empty");
1000template <
typename HashTableImpl>
1002 const unsigned char *Buckets,
const unsigned char *
const Payload,
1006 HashTable.reset(HashTableImpl::Create(
1007 Buckets, Payload,
Base,
1008 typename HashTableImpl::InfoType(HashType,
Version)));
1009 RecordIterator = HashTable->data_begin();
1012template <
typename HashTableImpl>
1024 : Underlying(Underlying) {}
1028 return Underlying.getRecords(FuncName, Data);
1034template <
typename HashTableImpl>
1039 std::unique_ptr<MemoryBuffer> RemapBuffer,
1041 : RemapBuffer(
std::
move(RemapBuffer)), Underlying(Underlying) {
1050 std::pair<StringRef, StringRef> Parts = {
StringRef(),
Name};
1053 if (Parts.first.starts_with(
"_Z"))
1055 if (Parts.second.empty())
1072 if (
Error E = Remappings.
read(*RemapBuffer))
1076 if (
auto Key = Remappings.
insert(RealName)) {
1080 MappedNames.insert({Key, RealName});
1089 if (
auto Key = Remappings.
lookup(RealName)) {
1090 StringRef Remapped = MappedNames.lookup(Key);
1091 if (!Remapped.
empty()) {
1093 RealName.
end() == FuncName.
end())
1094 FuncName = Remapped;
1099 Error E = Underlying.getRecords(Reconstituted,
Data);
1106 std::move(E), [](std::unique_ptr<InstrProfError> Err) {
1109 :
Error(std::move(Err));
1115 return Underlying.getRecords(FuncName,
Data);
1121 std::unique_ptr<MemoryBuffer> RemapBuffer;
1137 using namespace support;
1141 uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
1147const unsigned char *
1149 const unsigned char *Cur,
bool UseCS) {
1150 using namespace IndexedInstrProf;
1151 using namespace support;
1156 uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
1158 uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
1162 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
1167 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
1168 Dst[
I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[
I]);
1171 for (
unsigned I = 0;
I < SummaryData->NumCutoffEntries;
I++) {
1176 std::unique_ptr<llvm::ProfileSummary> &Summary =
1177 UseCS ? this->CS_Summary : this->Summary;
1180 Summary = std::make_unique<ProfileSummary>(
1182 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
1183 SummaryData->get(Summary::MaxBlockCount),
1184 SummaryData->get(Summary::MaxInternalBlockCount),
1185 SummaryData->get(Summary::MaxFunctionCount),
1186 SummaryData->get(Summary::TotalNumBlocks),
1187 SummaryData->get(Summary::TotalNumFunctions));
1188 return Cur + SummarySize;
1196 Summary = Builder.getSummary();
1201Error IndexedMemProfReader::deserializeV012(
const unsigned char *Start,
1202 const unsigned char *
Ptr,
1208 : support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1211 const uint64_t FramePayloadOffset =
1212 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1215 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1219 uint64_t CallStackPayloadOffset = 0;
1223 CallStackPayloadOffset =
1224 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1225 CallStackTableOffset =
1226 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1232 return SchemaOr.takeError();
1233 Schema = SchemaOr.get();
1237 Start + RecordTableOffset,
1243 Start + FrameTableOffset,
1244 Start + FramePayloadOffset,
1249 Start + CallStackTableOffset,
1250 Start + CallStackPayloadOffset,
1256Error IndexedMemProfReader::deserializeV3(
const unsigned char *Start,
1257 const unsigned char *
Ptr) {
1260 const uint64_t CallStackPayloadOffset =
1261 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1263 const uint64_t RecordPayloadOffset =
1264 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1267 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1272 return SchemaOr.takeError();
1273 Schema = SchemaOr.get();
1276 CallStackBase = Start + CallStackPayloadOffset;
1280 Start + RecordTableOffset,
1281 Start + RecordPayloadOffset,
1289 const unsigned char *
Ptr = Start + MemProfOffset;
1295 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1301 }
else if (FirstWord >= 24) {
1308 return make_error<InstrProfError>(
1310 formatv(
"MemProf version {} not supported; "
1311 "requires version between {} and {}, inclusive",
1320 if (
Error E = deserializeV012(Start,
Ptr, FirstWord))
1324 if (
Error E = deserializeV3(Start,
Ptr))
1329#ifdef EXPENSIVE_CHECKS
1337 for (
const auto &
Record : MemProfRecordTable->data())
1338 verifyIndexedMemProfRecord(
Record);
1345 using namespace support;
1347 const unsigned char *Start =
1348 (
const unsigned char *)DataBuffer->getBufferStart();
1349 const unsigned char *Cur = Start;
1350 if ((
const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
1355 return HeaderOr.takeError();
1358 Cur += Header->size();
1362 if (Header->Version & VARIANT_MASK_CSIR_PROF)
1372 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
1373 Start + Header->HashOffset, Cur, Start, HashType, Header->Version);
1377 if (Header->getIndexedProfileVersion() >= 8 &&
1378 Header->Version & VARIANT_MASK_MEMPROF) {
1385 if (Header->getIndexedProfileVersion() >= 9) {
1386 const unsigned char *
Ptr = Start + Header->BinaryIdOffset;
1389 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1390 if (BinaryIdsSize %
sizeof(
uint64_t))
1394 if (
Ptr > (
const unsigned char *)DataBuffer->getBufferEnd())
1396 "corrupted binary ids");
1399 if (Header->getIndexedProfileVersion() >= 12) {
1400 const unsigned char *
Ptr = Start + Header->VTableNamesOffset;
1402 uint64_t CompressedVTableNamesLen =
1403 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1407 const char *VTableNamePtr = (
const char *)
Ptr;
1408 if (VTableNamePtr > (
const char *)DataBuffer->getBufferEnd())
1411 VTableName =
StringRef(VTableNamePtr, CompressedVTableNamesLen);
1414 if (Header->getIndexedProfileVersion() >= 10 &&
1415 Header->Version & VARIANT_MASK_TEMPORAL_PROF) {
1416 const unsigned char *
Ptr = Start + Header->TemporalProfTracesOffset;
1417 const auto *PtrEnd = (
const unsigned char *)DataBuffer->getBufferEnd();
1422 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1424 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1425 for (
unsigned i = 0; i < NumTraces; i++) {
1431 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1433 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1435 if (
Ptr + NumFunctions *
sizeof(
uint64_t) > PtrEnd)
1437 for (
unsigned j = 0; j < NumFunctions; j++) {
1439 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1440 Trace.FunctionNameRefs.push_back(NameRef);
1447 if (RemappingBuffer) {
1449 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
1450 std::move(RemappingBuffer), *IndexPtr);
1451 if (
Error E = Remapper->populateRemappings())
1454 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1456 Index = std::move(IndexPtr);
1465 auto NewSymtab = std::make_unique<InstrProfSymtab>();
1467 if (
Error E = NewSymtab->initVTableNamesFromCompressedStrings(VTableName)) {
1473 if (
Error E =
Index->populateSymtab(*NewSymtab)) {
1478 Symtab = std::move(NewSymtab);
1487 auto Err = Remapper->getRecords(FuncName,
Data);
1494 return make_error<InstrProfError>(IE);
1495 if (
auto Err = Remapper->getRecords(DeprecatedFuncName,
Data))
1500 return std::move(Err2);
1506 bool CSBitMatch =
false;
1509 for (
uint64_t CountValue : Counts) {
1513 if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1514 return std::numeric_limits<uint64_t>::max();
1515 ValueSum += CountValue;
1522 if (
I.Hash == FuncHash)
1523 return std::move(
I);
1527 if (MismatchedFuncSum ==
nullptr)
1529 FuncSum = std::max(FuncSum, getFuncSum(
I.Counts));
1533 if (MismatchedFuncSum !=
nullptr)
1534 *MismatchedFuncSum = FuncSum;
1552 "memprof frame not found for frame id " +
1567 MemProfCallStackTable, FrameIdConv);
1573 return make_error<InstrProfError>(
1575 "memprof call stack not found for call stack id " +
1582 "memprof frame not found for frame id " +
1591 const unsigned char *FrameBase,
1592 const unsigned char *CallStackBase) {
1602 if (MemProfRecordTable ==
nullptr)
1604 "no memprof data available in profile");
1605 auto Iter = MemProfRecordTable->find(FuncNameHash);
1606 if (Iter == MemProfRecordTable->end())
1607 return make_error<InstrProfError>(
1609 "memprof record not found for function hash " +
Twine(FuncNameHash));
1615 assert(MemProfFrameTable &&
"MemProfFrameTable must be available");
1616 assert(!MemProfCallStackTable &&
1617 "MemProfCallStackTable must not be available");
1620 assert(MemProfFrameTable &&
"MemProfFrameTable must be available");
1621 assert(MemProfCallStackTable &&
"MemProfCallStackTable must be available");
1623 *MemProfCallStackTable);
1625 assert(!MemProfFrameTable &&
"MemProfFrameTable must not be available");
1626 assert(!MemProfCallStackTable &&
1627 "MemProfCallStackTable must not be available");
1628 assert(FrameBase &&
"FrameBase must be available");
1629 assert(CallStackBase &&
"CallStackBase must be available");
1633 return make_error<InstrProfError>(
1635 formatv(
"MemProf version {} not supported; "
1636 "requires version between {} and {}, inclusive",
1643 std::vector<uint64_t> &Counts) {
1646 return error(std::move(E));
1648 Counts =
Record.get().Counts;
1657 return error(std::move(E));
1659 const auto &BitmapBytes =
Record.get().BitmapBytes;
1660 size_t I = 0, E = BitmapBytes.size();
1661 Bitmap.
resize(E * CHAR_BIT);
1664 using XTy =
decltype(
X);
1665 alignas(XTy) uint8_t W[
sizeof(
X)];
1666 size_t N = std::min(E -
I,
sizeof(W));
1667 std::memset(W, 0,
sizeof(W));
1668 std::memcpy(W, &BitmapBytes[
I],
N);
1684 return error(std::move(E));
1687 if (RecordIndex >=
Data.size()) {
1688 Index->advanceToNextKey();
1695 std::vector<llvm::object::BuildID> &BinaryIds) {
1701 std::vector<llvm::object::BuildID> BinaryIds;
1710 for (
const auto &Func : *
this) {
1713 if (FuncIsCS != IsCS)
1716 Func.accumulateCounts(Sum);
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Provides ErrorOr<T> smart pointer.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
static Error initializeReader(InstrProfReader &Reader)
#define READ_NUM(Str, Dst)
static Expected< memprof::MemProfRecord > getMemProfRecordV0(const memprof::IndexedMemProfRecord &IndexedRecord, MemProfFrameHashTable &MemProfFrameTable)
#define CHECK_LINE_END(Line)
static Error readBinaryIdsInternal(const MemoryBuffer &DataBuffer, ArrayRef< uint8_t > BinaryIdsBuffer, std::vector< llvm::object::BuildID > &BinaryIds, const llvm::endianness Endian)
Read a list of binary ids from a profile that consist of a.
#define VP_READ_ADVANCE(Val)
static InstrProfKind getProfileKindFromVersion(uint64_t Version)
static Expected< memprof::MemProfRecord > getMemProfRecordV2(const memprof::IndexedMemProfRecord &IndexedRecord, MemProfFrameHashTable &MemProfFrameTable, MemProfCallStackHashTable &MemProfCallStackTable)
static Expected< memprof::MemProfRecord > getMemProfRecordV3(const memprof::IndexedMemProfRecord &IndexedRecord, const unsigned char *FrameBase, const unsigned char *CallStackBase)
static void printBinaryIdsInternal(raw_ostream &OS, ArrayRef< llvm::object::BuildID > BinaryIds)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
static BitVector & apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args)
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
reference get()
Returns a reference to the stored T value.
Reader for the indexed binary instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Error readHeader() override
Read the file header.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, BitVector &Bitmap)
Fill Bitmap with the profile data for the given function name.
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
Error deserialize(const unsigned char *Start, uint64_t MemProfOffset)
Expected< memprof::MemProfRecord > getMemProfRecord(const uint64_t FuncNameHash) const
InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.
const char * getNamesPointer() const
Return a pointer to the names string that this class constructs.
std::optional< size_t > getDataSize() const
Return the number of ProfileData elements.
size_t getNamesSize() const
Return the number of bytes in the names string.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
hash_value_type ComputeHash(StringRef K)
ArrayRef< NamedInstrProfRecord > data_type
InstrProfKind getProfileKind() const override
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
A remapper that applies remappings based on a symbol remapping file.
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
Error populateRemappings() override
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
Name matcher supporting fuzzy matching of symbol names to names in profiles.
Base class and interface for reading profiling data of any known instrprof format.
std::unique_ptr< InstrProfSymtab > Symtab
Error success()
Clear the current error and return a successful one.
SmallVector< TemporalProfTraceTy > TemporalProfTraces
A list of temporal profile traces.
uint64_t TemporalProfTraceStreamSize
The total number of temporal profile traces seen.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
virtual bool isIRLevelProfile() const =0
virtual Error readHeader()=0
Read the header. Required before reading first record.
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val)
Map the address range (i.e., [start_address, end_address)) of a variable to its names' MD5 hash.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
size_t getBufferSize() const
const char * getBufferEnd() const
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
const char * getBufferStart() const
Provides lookup and iteration over an on disk hash table.
static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())
Create the hash table.
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Reader for the raw instrprof binary format from runtime.
Error readHeader() override
Read the header. Required before reading first record.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
static bool hasFormat(const MemoryBuffer &DataBuffer)
InstrProfKind getProfileKind() const override
Returns a BitsetEnum describing the attributes of the raw instr profile.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
SmallVector< TemporalProfTraceTy > & getTemporalProfTraces(std::optional< uint64_t > Weight={}) override
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
iterator insert(iterator I, T &&Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr size_t size() const
size - Get the string size.
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Reader for symbol remapping files.
Key insert(StringRef FunctionName)
Construct a key for the given symbol, or return an existing one if an equivalent name has already bee...
Key lookup(StringRef FunctionName)
Map the given symbol name into the key for the corresponding equivalence class.
Error read(MemoryBuffer &B)
Read remappings from the given buffer, which must live as long as the remapper.
Reader for the simple text based instrprof format.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error readHeader() override
Read the header.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
LLVM Value Representation.
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
This class implements an extremely fast bulk output stream that can only output to a stream.
The virtual file system interface.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
constexpr uint64_t MaximumSupportedVersion
constexpr uint64_t MinimumSupportedVersion
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
value_type read(const void *memory, endianness endian)
Read a value of a particular endianness from memory.
This is an optimization pass for GlobalISel generic memory operations.
RawInstrProfReader< uint32_t > RawInstrProfReader32
constexpr T byteswap(T V) noexcept
Reverses the bytes in the given integer value V.
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
constexpr T alignToPowerOf2(U Value, V Align)
Will overflow only if result is not representable in T.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
std::vector< ProfileSummaryEntry > SummaryEntryVector
@ raw_profile_version_mismatch
@ counter_value_too_large
@ unexpected_correlation_info
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
constexpr char GlobalIdentifierDelimiter
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
RawInstrProfReader< uint64_t > RawInstrProfReader64
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
void consumeError(Error Err)
Consume a Error without doing anything.
InstrProfKind
An enum describing the attributes of an instrumented profile.
@ FunctionEntryInstrumentation
@ FrontendInstrumentation
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
uint64_t Cutoff
The required percentile of total execution count.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumSummaryFields
uint64_t NumCutoffEntries
Profiling information for a single function.
static bool hasCSFlagInHash(uint64_t FuncHash)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
std::optional< CallStackId > LastUnmappedId
std::optional< FrameId > LastUnmappedId
MemProfRecord toMemProfRecord(llvm::function_ref< std::vector< Frame >(const CallStackId)> Callback) const