34 #include <system_error>
44 if (
Version & VARIANT_MASK_IR_PROF) {
47 if (
Version & VARIANT_MASK_CSIR_PROF) {
50 if (
Version & VARIANT_MASK_INSTR_ENTRY) {
53 if (
Version & VARIANT_MASK_BYTE_COVERAGE) {
56 if (
Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
59 if (
Version & VARIANT_MASK_MEMPROF) {
69 if (std::error_code EC = BufferOrErr.
getError())
83 if (
Error E = BufferOrError.takeError())
95 if (Buffer->getBufferSize() == 0)
98 std::unique_ptr<InstrProfReader> Result;
122 if (
Error E = BufferOrError.takeError())
126 std::unique_ptr<MemoryBuffer> RemappingBuffer;
127 std::string RemappingPathStr = RemappingPath.
str();
128 if (!RemappingPathStr.empty()) {
130 if (
Error E = RemappingBufferOrError.takeError())
132 RemappingBuffer =
std::move(RemappingBufferOrError.get());
141 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
148 auto Result = std::make_unique<IndexedInstrProfReader>(
165 [](
char c) { return isPrint(c) || isSpace(c); });
176 if (Str.equals_insensitive(
"ir"))
178 else if (Str.equals_insensitive(
"fe"))
180 else if (Str.equals_insensitive(
"csir")) {
183 }
else if (Str.equals_insensitive(
"entry_first"))
185 else if (Str.equals_insensitive(
"not_entry_first"))
197 #define CHECK_LINE_END(Line) \
198 if (Line.is_at_end()) \
199 return error(instrprof_error::truncated);
200 #define READ_NUM(Str, Dst) \
201 if ((Str).getAsInteger(10, (Dst))) \
202 return error(instrprof_error::malformed);
203 #define VP_READ_ADVANCE(Val) \
204 CHECK_LINE_END(Line); \
206 READ_NUM((*Line), (Val)); \
217 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
219 "number of value kinds is invalid");
222 for (
uint32_t VK = 0; VK < NumValueKinds; VK++) {
231 Record.reserveSites(VK, NumValueSites);
235 std::vector<InstrProfValueData> CurrentValues;
236 for (
uint32_t V = 0; V < NumValueData; V++) {
238 std::pair<StringRef, StringRef> VD = Line->
rsplit(
':');
240 if (
ValueKind == IPVK_IndirectCallTarget) {
252 CurrentValues.push_back({
Value, TakenCount});
261 #undef CHECK_LINE_END
263 #undef VP_READ_ADVANCE
283 if ((Line++)->getAsInteger(0,
Record.Hash))
285 "function hash is not a valid integer");
291 if ((Line++)->getAsInteger(10, NumCounters))
293 "number of counters is not a valid integer");
294 if (NumCounters == 0)
299 Record.Counts.reserve(NumCounters);
304 if ((Line++)->getAsInteger(10, Count))
306 Record.Counts.push_back(Count);
316 template <
class IntPtrT>
321 template <
class IntPtrT>
327 return RawInstrProf::getMagic<IntPtrT>() ==
Magic ||
331 template <
class IntPtrT>
333 if (!hasFormat(*DataBuffer))
338 DataBuffer->getBufferStart());
339 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
340 return readHeader(*Header);
343 template <
class IntPtrT>
345 const char *End = DataBuffer->getBufferEnd();
347 while (CurrentPos != End && *CurrentPos == 0)
350 if (CurrentPos == End)
356 "not enough space for another header");
358 if (
reinterpret_cast<size_t>(CurrentPos) %
alignof(
uint64_t))
360 "insufficient padding");
363 if (
Magic !=
swap(RawInstrProf::getMagic<IntPtrT>()))
368 return readHeader(*Header);
371 template <
class IntPtrT>
376 const IntPtrT FPtr =
swap(
I->FunctionPointer);
384 template <
class IntPtrT>
390 if (useDebugInfoCorrelate() && !Correlator)
392 if (!useDebugInfoCorrelate() && Correlator)
395 BinaryIdsSize =
swap(Header.BinaryIdsSize);
396 if (BinaryIdsSize %
sizeof(
uint64_t))
399 CountersDelta =
swap(Header.CountersDelta);
400 NamesDelta =
swap(Header.NamesDelta);
401 auto NumData =
swap(Header.DataSize);
402 auto PaddingBytesBeforeCounters =
swap(Header.PaddingBytesBeforeCounters);
403 auto CountersSize =
swap(Header.CountersSize) * getCounterTypeSize();
404 auto PaddingBytesAfterCounters =
swap(Header.PaddingBytesAfterCounters);
405 auto NamesSize =
swap(Header.NamesSize);
406 ValueKindLast =
swap(Header.ValueKindLast);
409 auto PaddingSize = getNumPaddingBytes(NamesSize);
413 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
415 CountersOffset + CountersSize + PaddingBytesAfterCounters;
416 ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
418 auto *Start =
reinterpret_cast<const char *
>(&Header);
419 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
425 assert(DataSize == 0 && NamesSize == 0);
426 assert(CountersDelta == 0 && NamesDelta == 0);
427 Data = Correlator->getDataPointer();
428 DataEnd =
Data + Correlator->getDataSize();
429 NamesStart = Correlator->getNamesPointer();
430 NamesEnd = NamesStart + Correlator->getNamesSize();
434 DataEnd =
Data + NumData;
435 NamesStart = Start + NamesOffset;
436 NamesEnd = NamesStart + NamesSize;
442 CountersStart = Start + CountersOffset;
443 CountersEnd = CountersStart + CountersSize;
444 ValueDataStart =
reinterpret_cast<const uint8_t *
>(Start + ValueDataOffset);
446 const uint8_t *BufferEnd = (
const uint8_t *)DataBuffer->getBufferEnd();
447 if (BinaryIdsStart + BinaryIdsSize > BufferEnd)
450 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
451 if (
Error E = createSymtab(*NewSymtab))
458 template <
class IntPtrT>
464 template <
class IntPtrT>
470 template <
class IntPtrT>
474 if (NumCounters == 0)
478 if (CounterBaseOffset < 0)
481 (
"counter offset " +
Twine(CounterBaseOffset) +
" is negative").str());
483 if (CounterBaseOffset >= CountersEnd - CountersStart)
485 (
"counter offset " +
Twine(CounterBaseOffset) +
486 " is greater than the maximum counter offset " +
487 Twine(CountersEnd - CountersStart - 1))
491 (CountersEnd - (CountersStart + CounterBaseOffset)) /
492 getCounterTypeSize();
493 if (NumCounters > MaxNumCounters)
495 (
"number of counters " +
Twine(NumCounters) +
496 " is greater than the maximum number of counters " +
497 Twine(MaxNumCounters))
501 Record.Counts.reserve(NumCounters);
504 CountersStart + CounterBaseOffset +
I * getCounterTypeSize();
505 if (hasSingleByteCoverage()) {
507 Record.Counts.push_back(*Ptr == 0 ? 1 : 0);
509 const auto *CounterValue =
reinterpret_cast<const uint64_t *
>(Ptr);
517 template <
class IntPtrT>
521 CurValueDataSize = 0;
525 NumValueKinds += (
Data->NumValueSites[
I] != 0);
531 ValueProfData::getValueProfData(
532 ValueDataStart, (
const unsigned char *)DataBuffer->getBufferEnd(),
533 getDataEndianness());
541 VDataPtrOrErr.
get()->deserializeTo(
Record, Symtab.get());
542 CurValueDataSize = VDataPtrOrErr.
get()->getSize();
546 template <
class IntPtrT>
550 if (
Error E = readNextHeader(getNextHeaderPos()))
578 template <
class IntPtrT>
580 if (BinaryIdsSize == 0)
583 OS <<
"Binary IDs: \n";
584 const uint8_t *BI = BinaryIdsStart;
585 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
587 size_t Remaining = BIEnd - BI;
591 return make_error<InstrProfError>(
593 "not enough data to read binary id length");
599 if (Remaining <
sizeof(BinaryIdLen) + BinaryIdLen)
600 return make_error<InstrProfError>(
604 BI +=
sizeof(BinaryIdLen);
605 if (BI > (
const uint8_t *)DataBuffer->getBufferEnd())
606 return make_error<InstrProfError>(
608 "binary id that is read is bigger than buffer size");
617 if (BI > (
const uint8_t *)DataBuffer->getBufferEnd())
640 const unsigned char *&
D,
const unsigned char *
const End) {
642 ValueProfData::getValueProfData(
D, End, ValueProfDataEndianness);
647 VDataPtrOrErr.
get()->deserializeTo(DataBuffer.back(),
nullptr);
648 D += VDataPtrOrErr.
get()->TotalSize;
662 std::vector<uint64_t> CounterBuffer;
664 const unsigned char *End =
D +
N;
669 uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(
D);
677 CountsSize = endian::readNext<uint64_t, little, unaligned>(
D);
680 if (
D + CountsSize *
sizeof(
uint64_t) > End)
683 CounterBuffer.clear();
684 CounterBuffer.reserve(CountsSize);
685 for (
uint64_t J = 0; J < CountsSize; ++J)
686 CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(
D));
688 DataBuffer.emplace_back(K, Hash,
std::move(CounterBuffer));
700 template <
typename HashTableImpl>
703 auto Iter = HashTable->find(FuncName);
704 if (Iter == HashTable->end())
710 "profile data is empty");
715 template <
typename HashTableImpl>
721 Data = *RecordIterator;
725 "profile data is empty");
730 template <
typename HashTableImpl>
732 const unsigned char *Buckets,
const unsigned char *
const Payload,
736 HashTable.reset(HashTableImpl::Create(
737 Buckets, Payload,
Base,
739 RecordIterator = HashTable->data_begin();
742 template <
typename HashTableImpl>
764 template <
typename HashTableImpl>
769 std::unique_ptr<MemoryBuffer> RemapBuffer,
781 Parts = Parts.second.split(
':');
782 if (Parts.first.startswith(
"_Z"))
784 if (Parts.second.empty())
797 Out.
insert(Out.end(), ExtractedName.
end(), OrigName.
end());
801 if (
Error E = Remappings.read(*RemapBuffer))
805 if (
auto Key = Remappings.insert(RealName)) {
809 MappedNames.insert({
Key, RealName});
817 StringRef RealName = extractName(FuncName);
818 if (
auto Key = Remappings.lookup(RealName)) {
820 if (!Remapped.
empty()) {
822 RealName.
end() == FuncName.
end())
827 reconstituteName(FuncName, RealName, Remapped, Reconstituted);
835 std::move(
E), [](std::unique_ptr<InstrProfError> Err) {
850 std::unique_ptr<MemoryBuffer> RemapBuffer;
871 endian::read<uint64_t, little, aligned>(DataBuffer.
getBufferStart());
876 const unsigned char *
878 const unsigned char *Cur,
bool UseCS) {
879 using namespace IndexedInstrProf;
891 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
896 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
897 Dst[
I] = endian::byte_swap<uint64_t, little>(Src[
I]);
900 for (
unsigned I = 0;
I < SummaryData->NumCutoffEntries;
I++) {
905 std::unique_ptr<llvm::ProfileSummary> &Summary =
906 UseCS ? this->CS_Summary : this->Summary;
909 Summary = std::make_unique<ProfileSummary>(
911 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
912 SummaryData->get(Summary::MaxBlockCount),
913 SummaryData->get(Summary::MaxInternalBlockCount),
914 SummaryData->get(Summary::MaxFunctionCount),
915 SummaryData->get(Summary::TotalNumBlocks),
916 SummaryData->get(Summary::TotalNumFunctions));
917 return Cur + SummarySize;
925 Summary =
Builder.getSummary();
933 const unsigned char *Start =
934 (
const unsigned char *)DataBuffer->getBufferStart();
935 const unsigned char *Cur = Start;
936 if ((
const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
941 return HeaderOr.takeError();
944 Cur += Header->
size();
954 endian::byte_swap<uint64_t, little>(Header->
HashType));
961 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
962 Start + HashOffset, Cur, Start, HashType, Header->
formatVersion());
971 const unsigned char *Ptr = Start + MemProfOffset;
974 support::endian::readNext<uint64_t, little, unaligned>(Ptr);
977 support::endian::readNext<uint64_t, little, unaligned>(Ptr);
980 support::endian::readNext<uint64_t, little, unaligned>(Ptr);
985 return SchemaOr.takeError();
986 Schema = SchemaOr.get();
990 Start + RecordTableOffset,
996 Start + FrameTableOffset,
997 Start + FramePayloadOffset,
1002 if (RemappingBuffer) {
1003 Remapper = std::make_unique<
1006 if (
Error E = Remapper->populateRemappings())
1009 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1020 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
1021 if (
Error E =
Index->populateSymtab(*NewSymtab)) {
1033 Error Err = Remapper->getRecords(FuncName,
Data);
1039 if (
I.Hash == FuncHash)
1048 if (MemProfRecordTable ==
nullptr)
1050 "no memprof data available in profile");
1051 auto Iter = MemProfRecordTable->find(FuncNameHash);
1052 if (Iter == MemProfRecordTable->end())
1053 return make_error<InstrProfError>(
1055 "memprof record not found for function hash " +
Twine(FuncNameHash));
1060 bool HasFrameMappingError =
false;
1062 auto FrIter = MemProfFrameTable->find(
Id);
1063 if (FrIter == MemProfFrameTable->end()) {
1064 LastUnmappedFrameId =
Id;
1065 HasFrameMappingError =
true;
1074 if (HasFrameMappingError) {
1076 "memprof frame not found for frame id " +
1077 Twine(LastUnmappedFrameId));
1084 std::vector<uint64_t> &Counts) {
1089 Counts =
Record.get().Counts;
1101 if (RecordIndex >=
Data.size()) {
1102 Index->advanceToNextKey();
1110 for (
const auto &Func : *
this) {
1111 if (isIRLevelProfile()) {
1113 if (FuncIsCS != IsCS)
1116 Func.accumulateCounts(Sum);