35#include <system_error>
45 if (Version & VARIANT_MASK_IR_PROF) {
46 ProfileKind |= InstrProfKind::IRInstrumentation;
48 if (Version & VARIANT_MASK_CSIR_PROF) {
49 ProfileKind |= InstrProfKind::ContextSensitive;
51 if (Version & VARIANT_MASK_INSTR_ENTRY) {
52 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
54 if (Version & VARIANT_MASK_BYTE_COVERAGE) {
55 ProfileKind |= InstrProfKind::SingleByteCoverage;
57 if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
58 ProfileKind |= InstrProfKind::FunctionEntryOnly;
60 if (Version & VARIANT_MASK_MEMPROF) {
61 ProfileKind |= InstrProfKind::MemProf;
63 if (Version & VARIANT_MASK_TEMPORAL_PROF) {
64 ProfileKind |= InstrProfKind::TemporalProfile;
72 : FS.getBufferForFile(Filename);
73 if (std::error_code EC = BufferOrErr.getError())
75 return std::move(BufferOrErr.get());
93 const uint8_t *BinaryIdsStart,
94 std::vector<llvm::object::BuildID> &BinaryIds,
96 using namespace support;
98 if (BinaryIdsSize == 0)
101 const uint8_t *BI = BinaryIdsStart;
102 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
104 reinterpret_cast<const uint8_t *
>(DataBuffer.
getBufferEnd());
107 size_t Remaining = BIEnd - BI;
110 return make_error<InstrProfError>(
111 instrprof_error::malformed,
112 "not enough data to read binary id length");
117 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(BI);
119 BILen = endian::readNext<uint64_t, llvm::endianness::big, unaligned>(BI);
122 return make_error<InstrProfError>(instrprof_error::malformed,
123 "binary id length is 0");
125 Remaining = BIEnd - BI;
128 return make_error<InstrProfError>(
129 instrprof_error::malformed,
"not enough data to read binary id data");
137 return make_error<InstrProfError>(
138 instrprof_error::malformed,
139 "binary id section is greater than buffer size");
147 std::vector<llvm::object::BuildID> &BinaryIds) {
148 OS <<
"Binary IDs: \n";
149 for (
auto BI : BinaryIds) {
159 std::function<
void(
Error)> Warn) {
162 if (
Error E = BufferOrError.takeError())
171 std::function<
void(
Error)> Warn) {
172 if (Buffer->getBufferSize() == 0)
175 std::unique_ptr<InstrProfReader> Result;
192 return std::move(Result);
197 const Twine &RemappingPath) {
200 if (
Error E = BufferOrError.takeError())
204 std::unique_ptr<MemoryBuffer> RemappingBuffer;
205 std::string RemappingPathStr = RemappingPath.
str();
206 if (!RemappingPathStr.empty()) {
208 if (
Error E = RemappingBufferOrError.takeError())
210 RemappingBuffer = std::move(RemappingBufferOrError.get());
214 std::move(RemappingBuffer));
219 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
223 auto Result = std::make_unique<IndexedInstrProfReader>(
224 std::move(Buffer), std::move(RemappingBuffer));
230 return std::move(Result);
240 [](
char c) { return isPrint(c) || isSpace(c); });
251 if (Str.equals_insensitive(
"ir"))
253 else if (Str.equals_insensitive(
"fe"))
255 else if (Str.equals_insensitive(
"csir")) {
258 }
else if (Str.equals_insensitive(
"entry_first"))
260 else if (Str.equals_insensitive(
"not_entry_first"))
262 else if (Str.equals_insensitive(
"single_byte_coverage"))
264 else if (Str.equals_insensitive(
"temporal_prof_traces")) {
266 if (
auto Err = readTemporalProfTraceData())
267 return error(std::move(Err));
280Error TextInstrProfReader::readTemporalProfTraceData() {
281 if ((++Line).is_at_end())
288 if ((++Line).is_at_end())
294 for (
uint32_t i = 0; i < NumTraces; i++) {
295 if ((++Line).is_at_end())
302 if ((++Line).is_at_end())
306 Line->
split(FuncNames,
",", -1,
false);
307 for (
auto &FuncName : FuncNames)
308 Trace.FunctionNameRefs.push_back(
318#define CHECK_LINE_END(Line) \
319 if (Line.is_at_end()) \
320 return error(instrprof_error::truncated);
321#define READ_NUM(Str, Dst) \
322 if ((Str).getAsInteger(10, (Dst))) \
323 return error(instrprof_error::malformed);
324#define VP_READ_ADVANCE(Val) \
325 CHECK_LINE_END(Line); \
327 READ_NUM((*Line), (Val)); \
338 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
340 "number of value kinds is invalid");
343 for (
uint32_t VK = 0; VK < NumValueKinds; VK++) {
345 if (ValueKind > IPVK_Last)
352 Record.reserveSites(VK, NumValueSites);
353 for (
uint32_t S = 0; S < NumValueSites; S++) {
356 std::vector<InstrProfValueData> CurrentValues;
359 std::pair<StringRef, StringRef> VD = Line->
rsplit(
':');
361 if (ValueKind == IPVK_IndirectCallTarget) {
369 }
else if (ValueKind == IPVK_VTableTarget) {
378 CurrentValues.push_back({
Value, TakenCount});
381 Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
389#undef VP_READ_ADVANCE
404 return error(std::move(E));
409 if ((Line++)->getAsInteger(0,
Record.Hash))
411 "function hash is not a valid integer");
417 if ((Line++)->getAsInteger(10, NumCounters))
419 "number of counters is not a valid integer");
420 if (NumCounters == 0)
425 Record.Counts.reserve(NumCounters);
430 if ((Line++)->getAsInteger(10, Count))
432 Record.Counts.push_back(Count);
437 Record.BitmapBytes.clear();
440 if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes))
442 "number of bitmap bytes is not a valid integer");
443 if (NumBitmapBytes != 0) {
445 Record.BitmapBytes.reserve(NumBitmapBytes);
446 for (uint8_t
I = 0;
I < NumBitmapBytes; ++
I) {
450 if ((Line++)->getAsInteger(0, BitmapByte))
452 "bitmap byte is not a valid integer");
453 Record.BitmapBytes.push_back(BitmapByte);
460 return error(std::move(E));
465template <
class IntPtrT>
470template <
class IntPtrT>
473 std::optional<uint64_t> Weight) {
474 if (TemporalProfTimestamps.empty()) {
475 assert(TemporalProfTraces.empty());
476 return TemporalProfTraces;
479 std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end());
482 Trace.Weight = *Weight;
483 for (
auto &[TimestampValue, NameRef] : TemporalProfTimestamps)
484 Trace.FunctionNameRefs.push_back(NameRef);
485 TemporalProfTraces = {std::move(
Trace)};
486 return TemporalProfTraces;
489template <
class IntPtrT>
495 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
499template <
class IntPtrT>
501 if (!hasFormat(*DataBuffer))
506 DataBuffer->getBufferStart());
507 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
508 return readHeader(*Header);
511template <
class IntPtrT>
513 const char *
End = DataBuffer->getBufferEnd();
515 while (CurrentPos !=
End && *CurrentPos == 0)
518 if (CurrentPos ==
End)
524 "not enough space for another header");
526 if (
reinterpret_cast<size_t>(CurrentPos) %
alignof(
uint64_t))
528 "insufficient padding");
531 if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
536 return readHeader(*Header);
539template <
class IntPtrT>
542 return error(std::move(E));
544 const IntPtrT FPtr = swap(
I->FunctionPointer);
552template <
class IntPtrT>
558 (
"Profile uses raw profile format version = " +
559 Twine(GET_VERSION(Version)) +
561 "\nPLEASE update this tool to version in the raw profile, or "
562 "regenerate raw profile with expected version.")
567 const uint8_t *BinaryIdStart =
569 const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;
570 const uint8_t *BufferEnd = (
const uint8_t *)DataBuffer->getBufferEnd();
571 if (BinaryIdSize %
sizeof(
uint64_t) || BinaryIdEnd > BufferEnd)
573 if (BinaryIdSize != 0) {
576 BinaryIds, getDataEndianness()))
580 CountersDelta =
swap(Header.CountersDelta);
581 BitmapDelta =
swap(Header.BitmapDelta);
582 NamesDelta =
swap(Header.NamesDelta);
583 auto NumData =
swap(Header.NumData);
584 auto PaddingBytesBeforeCounters =
swap(Header.PaddingBytesBeforeCounters);
585 auto CountersSize =
swap(Header.NumCounters) * getCounterTypeSize();
586 auto PaddingBytesAfterCounters =
swap(Header.PaddingBytesAfterCounters);
587 auto NumBitmapBytes =
swap(Header.NumBitmapBytes);
588 auto PaddingBytesAfterBitmapBytes =
swap(Header.PaddingBytesAfterBitmapBytes);
589 auto NamesSize =
swap(Header.NamesSize);
590 auto VTableNameSize =
swap(Header.VNamesSize);
591 auto NumVTables =
swap(Header.NumVTables);
592 ValueKindLast =
swap(Header.ValueKindLast);
595 auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
596 auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
598 auto VTableSectionSize =
600 auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
604 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
606 CountersOffset + CountersSize + PaddingBytesAfterCounters;
608 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
610 NamesOffset + NamesSize + PaddingBytesAfterNames;
611 ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
612 PaddingBytesAfterVTableProfData;
614 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
616 auto *Start =
reinterpret_cast<const char *
>(&Header);
617 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
623 if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
626 Data = Correlator->getDataPointer();
633 DataEnd =
Data + NumData;
636 Start + VTableProfDataOffset);
637 VTableEnd = VTableBegin + NumVTables;
638 NamesStart = Start + NamesOffset;
639 NamesEnd = NamesStart + NamesSize;
640 VNamesStart = Start + VTableNameOffset;
641 VNamesEnd = VNamesStart + VTableNameSize;
644 CountersStart = Start + CountersOffset;
645 CountersEnd = CountersStart + CountersSize;
646 BitmapStart = Start + BitmapOffset;
647 BitmapEnd = BitmapStart + NumBitmapBytes;
648 ValueDataStart =
reinterpret_cast<const uint8_t *
>(Start + ValueDataOffset);
650 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
651 if (
Error E = createSymtab(*NewSymtab))
654 Symtab = std::move(NewSymtab);
658template <
class IntPtrT>
664template <
class IntPtrT>
670template <
class IntPtrT>
674 if (NumCounters == 0)
678 if (CounterBaseOffset < 0)
681 (
"counter offset " +
Twine(CounterBaseOffset) +
" is negative").str());
683 if (CounterBaseOffset >= CountersEnd - CountersStart)
685 (
"counter offset " +
Twine(CounterBaseOffset) +
686 " is greater than the maximum counter offset " +
687 Twine(CountersEnd - CountersStart - 1))
691 (CountersEnd - (CountersStart + CounterBaseOffset)) /
692 getCounterTypeSize();
693 if (NumCounters > MaxNumCounters)
695 (
"number of counters " +
Twine(NumCounters) +
696 " is greater than the maximum number of counters " +
697 Twine(MaxNumCounters))
701 Record.Counts.reserve(NumCounters);
704 CountersStart + CounterBaseOffset +
I * getCounterTypeSize();
705 if (
I == 0 && hasTemporalProfile()) {
707 if (TimestampValue != 0 &&
708 TimestampValue != std::numeric_limits<uint64_t>::max()) {
709 TemporalProfTimestamps.emplace_back(TimestampValue,
711 TemporalProfTraceStreamSize = 1;
713 if (hasSingleByteCoverage()) {
722 if (hasSingleByteCoverage()) {
724 Record.Counts.push_back(*
Ptr == 0 ? 1 : 0);
727 if (CounterValue > MaxCounterValue && Warn)
728 Warn(make_error<InstrProfError>(
731 Record.Counts.push_back(CounterValue);
738template <
class IntPtrT>
742 Record.BitmapBytes.clear();
743 Record.BitmapBytes.reserve(NumBitmapBytes);
747 if (NumBitmapBytes == 0)
752 if (BitmapOffset < 0)
755 (
"bitmap offset " +
Twine(BitmapOffset) +
" is negative").str());
757 if (BitmapOffset >= BitmapEnd - BitmapStart)
759 (
"bitmap offset " +
Twine(BitmapOffset) +
760 " is greater than the maximum bitmap offset " +
761 Twine(BitmapEnd - BitmapStart - 1))
765 (BitmapEnd - (BitmapStart + BitmapOffset)) /
sizeof(uint8_t);
766 if (NumBitmapBytes > MaxNumBitmapBytes)
768 (
"number of bitmap bytes " +
Twine(NumBitmapBytes) +
769 " is greater than the maximum number of bitmap bytes " +
770 Twine(MaxNumBitmapBytes))
774 const char *
Ptr = BitmapStart + BitmapOffset +
I;
781template <
class IntPtrT>
785 CurValueDataSize = 0;
789 NumValueKinds += (
Data->NumValueSites[
I] != 0);
795 ValueProfData::getValueProfData(
796 ValueDataStart, (
const unsigned char *)DataBuffer->getBufferEnd(),
797 getDataEndianness());
805 VDataPtrOrErr.
get()->deserializeTo(
Record, Symtab.get());
806 CurValueDataSize = VDataPtrOrErr.
get()->getSize();
810template <
class IntPtrT>
816 if (
Error E = readNextHeader(getNextHeaderPos()))
817 return error(std::move(E));
821 return error(std::move(E));
825 return error(std::move(E));
829 return error(std::move(E));
833 return error(std::move(E));
837 return error(std::move(E));
844template <
class IntPtrT>
846 std::vector<llvm::object::BuildID> &BinaryIds) {
847 BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(),
848 this->BinaryIds.end());
852template <
class IntPtrT>
854 if (!BinaryIds.empty())
875 const unsigned char *&
D,
const unsigned char *
const End) {
877 ValueProfData::getValueProfData(
D,
End, ValueProfDataEndianness);
882 VDataPtrOrErr.
get()->deserializeTo(DataBuffer.back(),
nullptr);
883 D += VDataPtrOrErr.
get()->TotalSize;
890 using namespace support;
897 std::vector<uint64_t> CounterBuffer;
898 std::vector<uint8_t> BitmapByteBuffer;
900 const unsigned char *
End =
D +
N;
906 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
D);
915 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
D);
921 CounterBuffer.clear();
922 CounterBuffer.reserve(CountsSize);
923 for (
uint64_t J = 0; J < CountsSize; ++J)
924 CounterBuffer.push_back(
925 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
D));
933 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
D);
935 if (
D + BitmapBytes *
sizeof(uint8_t) >
End)
937 BitmapByteBuffer.clear();
938 BitmapByteBuffer.reserve(BitmapBytes);
939 for (
uint64_t J = 0; J < BitmapBytes; ++J)
940 BitmapByteBuffer.push_back(
static_cast<uint8_t
>(
941 endian::readNext<uint64_t, llvm::endianness::little, unaligned>(
945 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer),
946 std::move(BitmapByteBuffer));
958template <
typename HashTableImpl>
961 auto Iter = HashTable->find(FuncName);
962 if (Iter == HashTable->end())
968 "profile data is empty");
973template <
typename HashTableImpl>
979 Data = *RecordIterator;
983 "profile data is empty");
988template <
typename HashTableImpl>
990 const unsigned char *Buckets,
const unsigned char *
const Payload,
993 FormatVersion = Version;
994 HashTable.reset(HashTableImpl::Create(
995 Buckets, Payload,
Base,
996 typename HashTableImpl::InfoType(HashType, Version)));
997 RecordIterator = HashTable->data_begin();
1000template <
typename HashTableImpl>
1012 : Underlying(Underlying) {}
1016 return Underlying.getRecords(FuncName, Data);
1022template <
typename HashTableImpl>
1027 std::unique_ptr<MemoryBuffer> RemapBuffer,
1029 : RemapBuffer(
std::
move(RemapBuffer)), Underlying(Underlying) {
1038 std::pair<StringRef, StringRef> Parts = {
StringRef(),
Name};
1041 if (Parts.first.starts_with(
"_Z"))
1043 if (Parts.second.empty())
1060 if (
Error E = Remappings.
read(*RemapBuffer))
1064 if (
auto Key = Remappings.
insert(RealName)) {
1068 MappedNames.insert({Key, RealName});
1077 if (
auto Key = Remappings.
lookup(RealName)) {
1078 StringRef Remapped = MappedNames.lookup(Key);
1079 if (!Remapped.
empty()) {
1081 RealName.
end() == FuncName.
end())
1082 FuncName = Remapped;
1087 Error E = Underlying.getRecords(Reconstituted,
Data);
1094 std::move(E), [](std::unique_ptr<InstrProfError> Err) {
1097 :
Error(std::move(Err));
1103 return Underlying.getRecords(FuncName,
Data);
1109 std::unique_ptr<MemoryBuffer> RemapBuffer;
1125 using namespace support;
1129 uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
1135const unsigned char *
1137 const unsigned char *Cur,
bool UseCS) {
1138 using namespace IndexedInstrProf;
1139 using namespace support;
1144 uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
1146 uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
1150 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
1155 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
1156 Dst[
I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[
I]);
1159 for (
unsigned I = 0;
I < SummaryData->NumCutoffEntries;
I++) {
1164 std::unique_ptr<llvm::ProfileSummary> &Summary =
1165 UseCS ? this->CS_Summary : this->Summary;
1168 Summary = std::make_unique<ProfileSummary>(
1170 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
1171 SummaryData->get(Summary::MaxBlockCount),
1172 SummaryData->get(Summary::MaxInternalBlockCount),
1173 SummaryData->get(Summary::MaxFunctionCount),
1174 SummaryData->get(Summary::TotalNumBlocks),
1175 SummaryData->get(Summary::TotalNumFunctions));
1176 return Cur + SummarySize;
1184 Summary = Builder.getSummary();
1190 using namespace support;
1192 const unsigned char *Start =
1193 (
const unsigned char *)DataBuffer->getBufferStart();
1194 const unsigned char *Cur = Start;
1195 if ((
const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
1200 return HeaderOr.takeError();
1203 Cur += Header->size();
1207 if (Header->formatVersion() & VARIANT_MASK_CSIR_PROF)
1213 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashType));
1218 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashOffset);
1221 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
1222 Start + HashOffset, Cur, Start, HashType, Header->formatVersion());
1226 if (GET_VERSION(Header->formatVersion()) >= 8 &&
1227 Header->formatVersion() & VARIANT_MASK_MEMPROF) {
1229 endian::byte_swap<uint64_t, llvm::endianness::little>(
1230 Header->MemProfOffset);
1232 const unsigned char *
Ptr = Start + MemProfOffset;
1239 const uint64_t FramePayloadOffset =
1250 return SchemaOr.takeError();
1251 Schema = SchemaOr.get();
1255 Start + RecordTableOffset,
1261 Start + FrameTableOffset,
1262 Start + FramePayloadOffset,
1265#ifdef EXPENSIVE_CHECKS
1269 for (
const auto &
Record : MemProfRecordTable->data())
1270 verifyIndexedMemProfRecord(
Record);
1276 if (GET_VERSION(Header->formatVersion()) >= 9) {
1278 endian::byte_swap<uint64_t, llvm::endianness::little>(
1279 Header->BinaryIdOffset);
1280 const unsigned char *
Ptr = Start + BinaryIdOffset;
1285 if (BinaryIdsSize %
sizeof(
uint64_t))
1288 BinaryIdsStart =
Ptr;
1289 if (BinaryIdsStart > (
const unsigned char *)DataBuffer->getBufferEnd())
1291 "corrupted binary ids");
1294 if (GET_VERSION(Header->formatVersion()) >= 12) {
1296 endian::byte_swap<uint64_t, llvm::endianness::little>(
1297 Header->VTableNamesOffset);
1298 const unsigned char *
Ptr = Start + VTableNamesOffset;
1300 CompressedVTableNamesLen =
1306 VTableNamePtr = (
const char *)
Ptr;
1307 if (VTableNamePtr > (
const char *)DataBuffer->getBufferEnd())
1311 if (GET_VERSION(Header->formatVersion()) >= 10 &&
1312 Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
1313 uint64_t TemporalProfTracesOffset =
1314 endian::byte_swap<uint64_t, llvm::endianness::little>(
1315 Header->TemporalProfTracesOffset);
1316 const unsigned char *
Ptr = Start + TemporalProfTracesOffset;
1317 const auto *PtrEnd = (
const unsigned char *)DataBuffer->getBufferEnd();
1327 for (
unsigned i = 0; i < NumTraces; i++) {
1339 if (
Ptr + NumFunctions *
sizeof(
uint64_t) > PtrEnd)
1341 for (
unsigned j = 0; j < NumFunctions; j++) {
1345 Trace.FunctionNameRefs.push_back(NameRef);
1352 if (RemappingBuffer) {
1354 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
1355 std::move(RemappingBuffer), *IndexPtr);
1356 if (
Error E = Remapper->populateRemappings())
1359 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1361 Index = std::move(IndexPtr);
1370 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
1371 if (
Error E =
Index->populateSymtab(*NewSymtab)) {
1376 Symtab = std::move(NewSymtab);
1385 auto Err = Remapper->getRecords(FuncName,
Data);
1392 return make_error<InstrProfError>(IE);
1393 if (
auto Err = Remapper->getRecords(DeprecatedFuncName,
Data))
1398 return std::move(Err2);
1404 bool CSBitMatch =
false;
1405 auto getFuncSum = [](
const std::vector<uint64_t> &Counts) {
1407 for (
uint64_t CountValue : Counts) {
1411 if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1412 return std::numeric_limits<uint64_t>::max();
1413 ValueSum += CountValue;
1420 if (
I.Hash == FuncHash)
1421 return std::move(
I);
1425 if (MismatchedFuncSum ==
nullptr)
1427 FuncSum = std::max(FuncSum, getFuncSum(
I.Counts));
1431 if (MismatchedFuncSum !=
nullptr)
1432 *MismatchedFuncSum = FuncSum;
1441 if (MemProfRecordTable ==
nullptr)
1443 "no memprof data available in profile");
1444 auto Iter = MemProfRecordTable->find(FuncNameHash);
1445 if (Iter == MemProfRecordTable->end())
1446 return make_error<InstrProfError>(
1448 "memprof record not found for function hash " +
Twine(FuncNameHash));
1453 bool HasFrameMappingError =
false;
1455 auto FrIter = MemProfFrameTable->find(Id);
1456 if (FrIter == MemProfFrameTable->end()) {
1457 LastUnmappedFrameId = Id;
1458 HasFrameMappingError =
true;
1467 if (HasFrameMappingError) {
1469 "memprof frame not found for frame id " +
1470 Twine(LastUnmappedFrameId));
1477 std::vector<uint64_t> &Counts) {
1480 return error(std::move(E));
1482 Counts =
Record.get().Counts;
1491 return error(std::move(E));
1493 const auto &BitmapBytes =
Record.get().BitmapBytes;
1494 size_t I = 0, E = BitmapBytes.size();
1495 Bitmap.
resize(E * CHAR_BIT);
1498 using XTy =
decltype(
X);
1499 alignas(XTy) uint8_t W[
sizeof(
X)];
1500 size_t N = std::min(E -
I,
sizeof(W));
1501 std::memset(W, 0,
sizeof(W));
1502 std::memcpy(W, &BitmapBytes[
I],
N);
1518 return error(std::move(E));
1521 if (RecordIndex >=
Data.size()) {
1522 Index->advanceToNextKey();
1529 std::vector<llvm::object::BuildID> &BinaryIds) {
1535 std::vector<llvm::object::BuildID> BinaryIds;
1544 for (
const auto &Func : *
this) {
1547 if (FuncIsCS != IsCS)
1550 Func.accumulateCounts(Sum);
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Provides ErrorOr<T> smart pointer.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
static Error initializeReader(InstrProfReader &Reader)
static void printBinaryIdsInternal(raw_ostream &OS, std::vector< llvm::object::BuildID > &BinaryIds)
static Error readBinaryIdsInternal(const MemoryBuffer &DataBuffer, const uint64_t BinaryIdsSize, const uint8_t *BinaryIdsStart, std::vector< llvm::object::BuildID > &BinaryIds, const llvm::endianness Endian)
Read a list of binary ids from a profile that consist of a.
#define READ_NUM(Str, Dst)
#define CHECK_LINE_END(Line)
#define VP_READ_ADVANCE(Val)
static InstrProfKind getProfileKindFromVersion(uint64_t Version)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
static BitVector & apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args)
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
reference get()
Returns a reference to the stored T value.
Reader for the indexed binary instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< memprof::MemProfRecord > getMemProfRecord(uint64_t FuncNameHash)
Return the memprof record for the function identified by llvm::md5(Name).
Error readHeader() override
Read the file header.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, BitVector &Bitmap)
Fill Bitmap with the profile data for the given function name.
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.
const char * getNamesPointer() const
Return a pointer to the names string that this class constructs.
std::optional< size_t > getDataSize() const
Return the number of ProfileData elements.
size_t getNamesSize() const
Return the number of bytes in the names string.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
hash_value_type ComputeHash(StringRef K)
ArrayRef< NamedInstrProfRecord > data_type
InstrProfKind getProfileKind() const override
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
A remapper that applies remappings based on a symbol remapping file.
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
Error populateRemappings() override
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
Name matcher supporting fuzzy matching of symbol names to names in profiles.
Base class and interface for reading profiling data of any known instrprof format.
std::unique_ptr< InstrProfSymtab > Symtab
Error success()
Clear the current error and return a successful one.
SmallVector< TemporalProfTraceTy > TemporalProfTraces
A list of temporal profile traces.
uint64_t TemporalProfTraceStreamSize
The total number of temporal profile traces seen.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
virtual bool isIRLevelProfile() const =0
virtual Error readHeader()=0
Read the header. Required before reading first record.
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
size_t getBufferSize() const
const char * getBufferEnd() const
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
const char * getBufferStart() const
static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())
Create the hash table.
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Reader for the raw instrprof binary format from runtime.
Error readHeader() override
Read the header. Required before reading first record.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
static bool hasFormat(const MemoryBuffer &DataBuffer)
InstrProfKind getProfileKind() const override
Returns a BitsetEnum describing the attributes of the raw instr profile.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
SmallVector< TemporalProfTraceTy > & getTemporalProfTraces(std::optional< uint64_t > Weight={}) override
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
iterator insert(iterator I, T &&Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr size_t size() const
size - Get the string size.
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Reader for symbol remapping files.
Key insert(StringRef FunctionName)
Construct a key for the given symbol, or return an existing one if an equivalent name has already bee...
Key lookup(StringRef FunctionName)
Map the given symbol name into the key for the corresponding equivalence class.
Error read(MemoryBuffer &B)
Read remappings from the given buffer, which must live as long as the remapper.
Reader for the simple text based instrprof format.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error readHeader() override
Read the header.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
LLVM Value Representation.
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
This class implements an extremely fast bulk output stream that can only output to a stream.
The virtual file system interface.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
value_type read(const void *memory, endianness endian)
Read a value of a particular endianness from memory.
value_type readNext(const CharT *&memory, endianness endian)
Read a value of a particular endianness from a buffer, and increment the buffer past that value.
This is an optimization pass for GlobalISel generic memory operations.
RawInstrProfReader< uint32_t > RawInstrProfReader32
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
constexpr T byteswap(T V) noexcept
Reverses the bytes in the given integer value V.
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
constexpr char kGlobalIdentifierDelimiter
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
std::vector< ProfileSummaryEntry > SummaryEntryVector
@ raw_profile_version_mismatch
@ counter_value_too_large
@ unexpected_correlation_info
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
RawInstrProfReader< uint64_t > RawInstrProfReader64
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
void consumeError(Error Err)
Consume a Error without doing anything.
InstrProfKind
An enum describing the attributes of an instrumented profile.
@ FunctionEntryInstrumentation
@ FrontendInstrumentation
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
uint64_t Cutoff
The required percentile of total execution count.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumSummaryFields
uint64_t NumCutoffEntries
Profiling information for a single function.
static bool hasCSFlagInHash(uint64_t FuncHash)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.