25 if (std::error_code EC = BufferOrErr.
getError())
27 return std::move(BufferOrErr.
get());
38 if (
Error E = BufferOrError.takeError())
46 if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max())
49 if (Buffer->getBufferSize() == 0)
52 std::unique_ptr<InstrProfReader> Result;
69 return std::move(Result);
76 if (
Error E = BufferOrError.takeError())
85 if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max())
91 auto Result = llvm::make_unique<IndexedInstrProfReader>(std::move(Buffer));
97 return std::move(Result);
100 void InstrProfIterator::Increment() {
115 [](
char c) { return ::isprint(c) || ::isspace(c); });
123 bool IsIRInstr =
false;
125 IsIRLevelProfile =
false;
137 IsIRLevelProfile = IsIRInstr;
144 #define CHECK_LINE_END(Line) \
145 if (Line.is_at_end()) \
146 return error(instrprof_error::truncated);
147 #define READ_NUM(Str, Dst) \
148 if ((Str).getAsInteger(10, (Dst))) \
149 return error(instrprof_error::malformed);
150 #define VP_READ_ADVANCE(Val) \
151 CHECK_LINE_END(Line); \
153 READ_NUM((*Line), (Val)); \
164 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
168 for (
uint32_t VK = 0; VK < NumValueKinds; VK++) {
170 if (ValueKind > IPVK_Last)
177 for (
uint32_t S = 0; S < NumValueSites; S++) {
180 std::vector<InstrProfValueData> CurrentValues;
181 for (
uint32_t V = 0; V < NumValueData; V++) {
183 std::pair<StringRef, StringRef> VD = Line->
rsplit(
':');
184 uint64_t TakenCount,
Value;
185 if (VK == IPVK_IndirectCallTarget) {
186 Symtab->addFuncName(VD.first);
192 CurrentValues.push_back({Value, TakenCount});
195 Record.
addValueData(VK, S, CurrentValues.data(), NumValueData,
nullptr);
200 #undef CHECK_LINE_END
202 #undef VP_READ_ADVANCE
216 Record.
Name = *Line++;
222 if ((Line++)->getAsInteger(0, Record.
Hash))
226 uint64_t NumCounters;
229 if ((Line++)->getAsInteger(10, NumCounters))
231 if (NumCounters == 0)
236 Record.
Counts.reserve(NumCounters);
237 for (uint64_t
I = 0;
I < NumCounters; ++
I) {
241 if ((Line++)->getAsInteger(10, Count))
243 Record.
Counts.push_back(Count);
247 if (
Error E = readValueProfileData(Record))
256 template <
class IntPtrT>
262 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
266 template <
class IntPtrT>
268 if (!hasFormat(*DataBuffer))
273 DataBuffer->getBufferStart());
274 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
275 return readHeader(*Header);
278 template <
class IntPtrT>
280 const char *
End = DataBuffer->getBufferEnd();
282 while (CurrentPos != End && *CurrentPos == 0)
285 if (CurrentPos == End)
292 if (reinterpret_cast<size_t>(CurrentPos) %
alignof(uint64_t))
295 uint64_t
Magic = *
reinterpret_cast<const uint64_t *
>(CurrentPos);
296 if (Magic !=
swap(RawInstrProf::getMagic<IntPtrT>()))
301 return readHeader(*Header);
304 template <
class IntPtrT>
307 return error(std::move(
E));
308 for (
const RawInstrProf::ProfileData<IntPtrT> *
I = Data;
I != DataEnd; ++
I) {
309 const IntPtrT FPtr =
swap(
I->FunctionPointer);
318 template <
class IntPtrT>
325 CountersDelta =
swap(Header.CountersDelta);
326 NamesDelta =
swap(Header.NamesDelta);
327 auto DataSize =
swap(Header.DataSize);
328 auto CountersSize =
swap(Header.CountersSize);
329 NamesSize =
swap(Header.NamesSize);
330 ValueKindLast =
swap(Header.ValueKindLast);
332 auto DataSizeInBytes = DataSize *
sizeof(RawInstrProf::ProfileData<IntPtrT>);
333 auto PaddingSize = getNumPaddingBytes(NamesSize);
336 ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes;
337 ptrdiff_t NamesOffset = CountersOffset +
sizeof(uint64_t) * CountersSize;
338 ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
340 auto *Start =
reinterpret_cast<const char *
>(&Header);
341 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
344 Data =
reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *
>(
346 DataEnd =
Data + DataSize;
347 CountersStart =
reinterpret_cast<const uint64_t *
>(Start + CountersOffset);
348 NamesStart = Start + NamesOffset;
349 ValueDataStart =
reinterpret_cast<const uint8_t *
>(Start + ValueDataOffset);
351 std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>();
352 if (
Error E = createSymtab(*NewSymtab.get()))
355 Symtab = std::move(NewSymtab);
359 template <
class IntPtrT>
365 template <
class IntPtrT>
371 template <
class IntPtrT>
375 IntPtrT CounterPtr =
Data->CounterPtr;
376 if (NumCounters == 0)
379 auto RawCounts =
makeArrayRef(getCounter(CounterPtr), NumCounters);
380 auto *NamesStartAsCounter =
reinterpret_cast<const uint64_t *
>(NamesStart);
383 if (RawCounts.data() < CountersStart ||
384 RawCounts.data() + RawCounts.size() > NamesStartAsCounter)
387 if (ShouldSwapBytes) {
389 Record.
Counts.reserve(RawCounts.size());
390 for (uint64_t Count : RawCounts)
393 Record.
Counts = RawCounts;
398 template <
class IntPtrT>
403 CurValueDataSize = 0;
407 NumValueKinds += (
Data->NumValueSites[
I] != 0);
413 ValueProfData::getValueProfData(
414 ValueDataStart, (
const unsigned char *)DataBuffer->getBufferEnd(),
415 getDataEndianness());
424 CurValueDataSize = VDataPtrOrErr.
get()->getSize();
428 template <
class IntPtrT>
432 if (
Error E = readNextHeader(getNextHeaderPos()))
436 if (
Error E = readName(Record))
440 if (
Error E = readFuncHash(Record))
444 if (
Error E = readRawCounts(Record))
448 if (
Error E = readValueProfilingData(Record))
470 const unsigned char *&
D,
const unsigned char *
const End) {
472 ValueProfData::getValueProfData(D, End, ValueProfDataEndianness);
477 VDataPtrOrErr.
get()->deserializeTo(DataBuffer.back(),
nullptr);
478 D += VDataPtrOrErr.
get()->TotalSize;
486 if (N %
sizeof(uint64_t))
490 std::vector<uint64_t> CounterBuffer;
492 using namespace support;
493 const unsigned char *End = D +
N;
496 if (D +
sizeof(uint64_t) >= End)
498 uint64_t
Hash = endian::readNext<uint64_t, little, unaligned>(
D);
501 uint64_t CountsSize = N /
sizeof(uint64_t) - 1;
504 if (D +
sizeof(uint64_t) >
End)
506 CountsSize = endian::readNext<uint64_t, little, unaligned>(
D);
509 if (D + CountsSize *
sizeof(uint64_t) >
End)
512 CounterBuffer.clear();
513 CounterBuffer.reserve(CountsSize);
514 for (uint64_t J = 0; J < CountsSize; ++J)
515 CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
517 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
529 template <
typename HashTableImpl>
532 auto Iter = HashTable->find(FuncName);
533 if (Iter == HashTable->end())
543 template <
typename HashTableImpl>
549 Data = *RecordIterator;
557 template <
typename HashTableImpl>
559 const unsigned char *Buckets,
const unsigned char *
const Payload,
563 HashTable.reset(HashTableImpl::Create(
564 Buckets, Payload, Base,
565 typename HashTableImpl::InfoType(HashType, Version)));
566 RecordIterator = HashTable->data_begin();
572 using namespace support;
574 endian::read<uint64_t, little, aligned>(DataBuffer.
getBufferStart());
579 const unsigned char *
581 const unsigned char *Cur) {
582 using namespace IndexedInstrProf;
583 using namespace support;
593 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
596 const uint64_t *Src =
reinterpret_cast<const uint64_t *
>(SummaryInLE);
597 uint64_t *Dst =
reinterpret_cast<uint64_t *
>(SummaryData.get());
598 for (
unsigned I = 0;
I < SummarySize /
sizeof(uint64_t);
I++)
599 Dst[
I] = endian::byte_swap<uint64_t, little>(Src[
I]);
602 for (
unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
608 this->Summary = llvm::make_unique<ProfileSummary>(
610 SummaryData->get(Summary::TotalBlockCount),
611 SummaryData->get(Summary::MaxBlockCount),
612 SummaryData->get(Summary::MaxInternalBlockCount),
613 SummaryData->get(Summary::MaxFunctionCount),
614 SummaryData->get(Summary::TotalNumBlocks),
615 SummaryData->get(Summary::TotalNumFunctions));
616 return Cur + SummarySize;
619 using namespace IndexedInstrProf;
623 this->Summary = Builder.getSummary();
629 const unsigned char *Start =
630 (
const unsigned char *)DataBuffer->getBufferStart();
631 const unsigned char *Cur = Start;
632 if ((
const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
635 using namespace support;
641 uint64_t Magic = endian::byte_swap<uint64_t, little>(Header->Magic);
646 uint64_t FormatVersion = endian::byte_swap<uint64_t, little>(Header->Version);
647 if (GET_VERSION(FormatVersion) >
655 endian::byte_swap<uint64_t, little>(Header->HashType));
659 uint64_t HashOffset = endian::byte_swap<uint64_t, little>(Header->HashOffset);
664 Start + HashOffset, Cur, Start, HashType, FormatVersion);
665 Index.reset(IndexPtr);
671 return *Symtab.get();
673 std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>();
674 Index->populateSymtab(*NewSymtab.get());
676 Symtab = std::move(NewSymtab);
677 return *Symtab.get();
684 Error Err = Index->getRecords(FuncName, Data);
686 return std::move(Err);
688 for (
unsigned I = 0,
E = Data.
size(); I <
E; ++
I) {
690 if (Data[I].
Hash == FuncHash) {
691 return std::move(Data[I]);
699 std::vector<uint64_t> &Counts) {
702 return error(std::move(
E));
704 Counts = Record.
get().Counts;
709 static unsigned RecordIndex = 0;
713 Error E = Index->getRecords(Data);
715 return error(std::move(E));
717 Record = Data[RecordIndex++];
718 if (RecordIndex >= Data.
size()) {
719 Index->advanceToNextKey();
Error getRecords(ArrayRef< InstrProfRecord > &Data) override
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
#define VP_READ_ADVANCE(Val)
std::error_code getError() const
Error readNextRecord(InstrProfRecord &Record) override
Read a single record.
Represents either an error or a value T.
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Error readHeader() override
Read the header. Required before reading first record.
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumCutoffEntries
const char * getBufferStart() const
uint64_t MinBlockCount
The minimum execution count for this percentile.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
static instrprof_error take(Error E)
Consume an Error and return the raw enum value contained within it.
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Error error(instrprof_error Err)
Set the current error and return same.
AddrHashMap & getAddrHashMap()
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
Error takeError()
Take ownership of the stored error.
Error readNextRecord(InstrProfRecord &Record) override
Read a single record.
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
Error readHeader() override
Read the header.
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
virtual Error readNextRecord(InstrProfRecord &Record)=0
Read a single record.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path)
Factory method to create an appropriately typed reader for the given instrprof file.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
InstrProfLookupTrait::data_type data_type
static StringRef getName(Value *V)
RawInstrProfReader< uint32_t > RawInstrProfReader32
Tagged union holding either a T or a Error.
InstrProfLookupTrait::offset_type offset_type
uint64_t NumSummaryFields
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Reader for the simple text based instrprof format.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
size_t size() const
size - Get the array size.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
hash_value_type ComputeHash(StringRef K)
Base class and interface for reading profiling data of any known instrprof format.
uint64_t Cutoff
The required percentile of total execution count.
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
LLVM_NODISCARD std::pair< StringRef, StringRef > rsplit(char Separator) const
Split into two substrings around the last occurrence of a separator character.
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
std::unique_ptr< InstrProfSymtab > Symtab
static const unsigned End
bool empty() const
empty - Check if the array is empty.
uint64_t ComputeHash(StringRef K)
#define CHECK_LINE_END(Line)
#define READ_NUM(Str, Dst)
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, ValueMapType *ValueMap)
Add ValueData for ValueKind at value Site.
void clearValueData()
Clear value data entries.
static const char *const Magic
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Path)
static ErrorSuccess success()
Create a success value.
Reader for the raw instrprof binary format from runtime.
reference get()
Returns a reference to the stored T value.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
unsigned char getSwappedBytes(unsigned char C)
size_t getBufferSize() const
success
Parameters (see the expansion example below): (the builder, addr, loaded, new_val, ordering, /* OUT.
RawInstrProfReader< uint64_t > RawInstrProfReader64
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, int64_t FileSize=-1, bool RequiresNullTerminator=true)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
std::vector< uint64_t > Counts
Error readHeader() override
Read the file header.
Error readNextRecord(InstrProfRecord &Record) override
Read a single record.
virtual Error readHeader()=0
Read the header. Required before reading first record.
Profiling information for a single function.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash)
Return the pointer to InstrProfRecord associated with FuncName and FuncHash.
ArrayRef< InstrProfRecord > data_type
LLVM Value Representation.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path)
Factory method to create an indexed reader.
Lightweight error class with error context and mandatory checking.
std::string Hash(const Unit &U)
StringRef - Represent a constant reference to a string, i.e.
Reader for the indexed binary instrprof format.
Error success()
Clear the current error and return a successful one.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
std::vector< ProfileSummaryEntry > SummaryEntryVector
static Error initializeReader(InstrProfReader &Reader)