LLVM  10.0.0svn
InstrProfReader.cpp
Go to the documentation of this file.
1 //===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains support for reading profiling data for clang's
10 // instrumentation based PGO and coverage.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/IR/ProfileSummary.h"
22 #include "llvm/Support/Endian.h"
23 #include "llvm/Support/Error.h"
24 #include "llvm/Support/ErrorOr.h"
28 #include <algorithm>
29 #include <cctype>
30 #include <cstddef>
31 #include <cstdint>
32 #include <limits>
33 #include <memory>
34 #include <system_error>
35 #include <utility>
36 #include <vector>
37 
38 using namespace llvm;
39 
41 setupMemoryBuffer(const Twine &Path) {
44  if (std::error_code EC = BufferOrErr.getError())
45  return errorCodeToError(EC);
46  return std::move(BufferOrErr.get());
47 }
48 
50  return Reader.readHeader();
51 }
52 
55  // Set up the buffer to read.
56  auto BufferOrError = setupMemoryBuffer(Path);
57  if (Error E = BufferOrError.takeError())
58  return std::move(E);
59  return InstrProfReader::create(std::move(BufferOrError.get()));
60 }
61 
63 InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
64  // Sanity check the buffer.
65  if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint64_t>::max())
66  return make_error<InstrProfError>(instrprof_error::too_large);
67 
68  if (Buffer->getBufferSize() == 0)
69  return make_error<InstrProfError>(instrprof_error::empty_raw_profile);
70 
71  std::unique_ptr<InstrProfReader> Result;
72  // Create the reader.
74  Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
75  else if (RawInstrProfReader64::hasFormat(*Buffer))
76  Result.reset(new RawInstrProfReader64(std::move(Buffer)));
77  else if (RawInstrProfReader32::hasFormat(*Buffer))
78  Result.reset(new RawInstrProfReader32(std::move(Buffer)));
79  else if (TextInstrProfReader::hasFormat(*Buffer))
80  Result.reset(new TextInstrProfReader(std::move(Buffer)));
81  else
82  return make_error<InstrProfError>(instrprof_error::unrecognized_format);
83 
84  // Initialize the reader and return the result.
85  if (Error E = initializeReader(*Result))
86  return std::move(E);
87 
88  return std::move(Result);
89 }
90 
92 IndexedInstrProfReader::create(const Twine &Path, const Twine &RemappingPath) {
93  // Set up the buffer to read.
94  auto BufferOrError = setupMemoryBuffer(Path);
95  if (Error E = BufferOrError.takeError())
96  return std::move(E);
97 
98  // Set up the remapping buffer if requested.
99  std::unique_ptr<MemoryBuffer> RemappingBuffer;
100  std::string RemappingPathStr = RemappingPath.str();
101  if (!RemappingPathStr.empty()) {
102  auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr);
103  if (Error E = RemappingBufferOrError.takeError())
104  return std::move(E);
105  RemappingBuffer = std::move(RemappingBufferOrError.get());
106  }
107 
108  return IndexedInstrProfReader::create(std::move(BufferOrError.get()),
109  std::move(RemappingBuffer));
110 }
111 
113 IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
114  std::unique_ptr<MemoryBuffer> RemappingBuffer) {
115  // Sanity check the buffer.
116  if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint64_t>::max())
117  return make_error<InstrProfError>(instrprof_error::too_large);
118 
119  // Create the reader.
120  if (!IndexedInstrProfReader::hasFormat(*Buffer))
121  return make_error<InstrProfError>(instrprof_error::bad_magic);
122  auto Result = std::make_unique<IndexedInstrProfReader>(
123  std::move(Buffer), std::move(RemappingBuffer));
124 
125  // Initialize the reader and return the result.
126  if (Error E = initializeReader(*Result))
127  return std::move(E);
128 
129  return std::move(Result);
130 }
131 
132 void InstrProfIterator::Increment() {
133  if (auto E = Reader->readNextRecord(Record)) {
134  // Handle errors in the reader.
135  InstrProfError::take(std::move(E));
136  *this = InstrProfIterator();
137  }
138 }
139 
141  // Verify that this really looks like plain ASCII text by checking a
142  // 'reasonable' number of characters (up to profile magic size).
143  size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t));
144  StringRef buffer = Buffer.getBufferStart();
145  return count == 0 ||
146  std::all_of(buffer.begin(), buffer.begin() + count,
147  [](char c) { return isPrint(c) || ::isspace(c); });
148 }
149 
150 // Read the profile variant flag from the header: ":FE" means this is a FE
151 // generated profile. ":IR" means this is an IR level profile. Other strings
152 // with a leading ':' will be reported an error format.
154  Symtab.reset(new InstrProfSymtab());
155  bool IsIRInstr = false;
156  if (!Line->startswith(":")) {
157  IsIRLevelProfile = false;
158  return success();
159  }
160  StringRef Str = (Line)->substr(1);
161  if (Str.equals_lower("ir"))
162  IsIRInstr = true;
163  else if (Str.equals_lower("fe"))
164  IsIRInstr = false;
165  else if (Str.equals_lower("csir")) {
166  IsIRInstr = true;
167  HasCSIRLevelProfile = true;
168  } else
170 
171  ++Line;
172  IsIRLevelProfile = IsIRInstr;
173  return success();
174 }
175 
176 Error
177 TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
178 
179 #define CHECK_LINE_END(Line) \
180  if (Line.is_at_end()) \
181  return error(instrprof_error::truncated);
182 #define READ_NUM(Str, Dst) \
183  if ((Str).getAsInteger(10, (Dst))) \
184  return error(instrprof_error::malformed);
185 #define VP_READ_ADVANCE(Val) \
186  CHECK_LINE_END(Line); \
187  uint32_t Val; \
188  READ_NUM((*Line), (Val)); \
189  Line++;
190 
191  if (Line.is_at_end())
192  return success();
193 
194  uint32_t NumValueKinds;
195  if (Line->getAsInteger(10, NumValueKinds)) {
196  // No value profile data
197  return success();
198  }
199  if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
201  Line++;
202 
203  for (uint32_t VK = 0; VK < NumValueKinds; VK++) {
205  if (ValueKind > IPVK_Last)
207  VP_READ_ADVANCE(NumValueSites);
208  if (!NumValueSites)
209  continue;
210 
211  Record.reserveSites(VK, NumValueSites);
212  for (uint32_t S = 0; S < NumValueSites; S++) {
213  VP_READ_ADVANCE(NumValueData);
214 
215  std::vector<InstrProfValueData> CurrentValues;
216  for (uint32_t V = 0; V < NumValueData; V++) {
217  CHECK_LINE_END(Line);
218  std::pair<StringRef, StringRef> VD = Line->rsplit(':');
219  uint64_t TakenCount, Value;
220  if (ValueKind == IPVK_IndirectCallTarget) {
221  if (InstrProfSymtab::isExternalSymbol(VD.first)) {
222  Value = 0;
223  } else {
224  if (Error E = Symtab->addFuncName(VD.first))
225  return E;
226  Value = IndexedInstrProf::ComputeHash(VD.first);
227  }
228  } else {
229  READ_NUM(VD.first, Value);
230  }
231  READ_NUM(VD.second, TakenCount);
232  CurrentValues.push_back({Value, TakenCount});
233  Line++;
234  }
235  Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
236  nullptr);
237  }
238  }
239  return success();
240 
241 #undef CHECK_LINE_END
242 #undef READ_NUM
243 #undef VP_READ_ADVANCE
244 }
245 
247  // Skip empty lines and comments.
248  while (!Line.is_at_end() && (Line->empty() || Line->startswith("#")))
249  ++Line;
250  // If we hit EOF while looking for a name, we're done.
251  if (Line.is_at_end()) {
252  return error(instrprof_error::eof);
253  }
254 
255  // Read the function name.
256  Record.Name = *Line++;
257  if (Error E = Symtab->addFuncName(Record.Name))
258  return error(std::move(E));
259 
260  // Read the function hash.
261  if (Line.is_at_end())
263  if ((Line++)->getAsInteger(0, Record.Hash))
265 
266  // Read the number of counters.
267  uint64_t NumCounters;
268  if (Line.is_at_end())
270  if ((Line++)->getAsInteger(10, NumCounters))
272  if (NumCounters == 0)
274 
275  // Read each counter and fill our internal storage with the values.
276  Record.Clear();
277  Record.Counts.reserve(NumCounters);
278  for (uint64_t I = 0; I < NumCounters; ++I) {
279  if (Line.is_at_end())
281  uint64_t Count;
282  if ((Line++)->getAsInteger(10, Count))
284  Record.Counts.push_back(Count);
285  }
286 
287  // Check if value profile data exists and read it if so.
288  if (Error E = readValueProfileData(Record))
289  return error(std::move(E));
290 
291  return success();
292 }
293 
294 template <class IntPtrT>
296  if (DataBuffer.getBufferSize() < sizeof(uint64_t))
297  return false;
298  uint64_t Magic =
299  *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
300  return RawInstrProf::getMagic<IntPtrT>() == Magic ||
301  sys::getSwappedBytes(RawInstrProf::getMagic<IntPtrT>()) == Magic;
302 }
303 
304 template <class IntPtrT>
306  if (!hasFormat(*DataBuffer))
308  if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
310  auto *Header = reinterpret_cast<const RawInstrProf::Header *>(
311  DataBuffer->getBufferStart());
312  ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
313  return readHeader(*Header);
314 }
315 
316 template <class IntPtrT>
317 Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
318  const char *End = DataBuffer->getBufferEnd();
319  // Skip zero padding between profiles.
320  while (CurrentPos != End && *CurrentPos == 0)
321  ++CurrentPos;
322  // If there's nothing left, we're done.
323  if (CurrentPos == End)
324  return make_error<InstrProfError>(instrprof_error::eof);
325  // If there isn't enough space for another header, this is probably just
326  // garbage at the end of the file.
327  if (CurrentPos + sizeof(RawInstrProf::Header) > End)
328  return make_error<InstrProfError>(instrprof_error::malformed);
329  // The writer ensures each profile is padded to start at an aligned address.
330  if (reinterpret_cast<size_t>(CurrentPos) % alignof(uint64_t))
331  return make_error<InstrProfError>(instrprof_error::malformed);
332  // The magic should have the same byte order as in the previous header.
333  uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
334  if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
335  return make_error<InstrProfError>(instrprof_error::bad_magic);
336 
337  // There's another profile to read, so we need to process the header.
338  auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
339  return readHeader(*Header);
340 }
341 
342 template <class IntPtrT>
344  if (Error E = Symtab.create(StringRef(NamesStart, NamesSize)))
345  return error(std::move(E));
346  for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
347  const IntPtrT FPtr = swap(I->FunctionPointer);
348  if (!FPtr)
349  continue;
350  Symtab.mapAddress(FPtr, I->NameRef);
351  }
352  return success();
353 }
354 
355 template <class IntPtrT>
357  const RawInstrProf::Header &Header) {
358  Version = swap(Header.Version);
359  if (GET_VERSION(Version) != RawInstrProf::Version)
361 
362  CountersDelta = swap(Header.CountersDelta);
363  NamesDelta = swap(Header.NamesDelta);
364  auto DataSize = swap(Header.DataSize);
365  auto CountersSize = swap(Header.CountersSize);
366  NamesSize = swap(Header.NamesSize);
367  ValueKindLast = swap(Header.ValueKindLast);
368 
369  auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>);
370  auto PaddingSize = getNumPaddingBytes(NamesSize);
371 
372  ptrdiff_t DataOffset = sizeof(RawInstrProf::Header);
373  ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes;
374  ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize;
375  ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
376 
377  auto *Start = reinterpret_cast<const char *>(&Header);
378  if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
380 
381  Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
382  Start + DataOffset);
383  DataEnd = Data + DataSize;
384  CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset);
385  NamesStart = Start + NamesOffset;
386  ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
387 
388  std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
389  if (Error E = createSymtab(*NewSymtab.get()))
390  return E;
391 
392  Symtab = std::move(NewSymtab);
393  return success();
394 }
395 
396 template <class IntPtrT>
398  Record.Name = getName(Data->NameRef);
399  return success();
400 }
401 
402 template <class IntPtrT>
404  Record.Hash = swap(Data->FuncHash);
405  return success();
406 }
407 
408 template <class IntPtrT>
410  InstrProfRecord &Record) {
411  uint32_t NumCounters = swap(Data->NumCounters);
412  IntPtrT CounterPtr = Data->CounterPtr;
413  if (NumCounters == 0)
415 
416  auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart);
417  ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart;
418 
419  // Check bounds. Note that the counter pointer embedded in the data record
420  // may itself be corrupt.
421  if (NumCounters > MaxNumCounters)
423  ptrdiff_t CounterOffset = getCounterOffset(CounterPtr);
424  if (CounterOffset < 0 || CounterOffset > MaxNumCounters ||
425  (CounterOffset + NumCounters) > MaxNumCounters)
427 
428  auto RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters);
429 
430  if (ShouldSwapBytes) {
431  Record.Counts.clear();
432  Record.Counts.reserve(RawCounts.size());
433  for (uint64_t Count : RawCounts)
434  Record.Counts.push_back(swap(Count));
435  } else
436  Record.Counts = RawCounts;
437 
438  return success();
439 }
440 
441 template <class IntPtrT>
443  InstrProfRecord &Record) {
444  Record.clearValueData();
445  CurValueDataSize = 0;
446  // Need to match the logic in value profile dumper code in compiler-rt:
447  uint32_t NumValueKinds = 0;
448  for (uint32_t I = 0; I < IPVK_Last + 1; I++)
449  NumValueKinds += (Data->NumValueSites[I] != 0);
450 
451  if (!NumValueKinds)
452  return success();
453 
455  ValueProfData::getValueProfData(
456  ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(),
457  getDataEndianness());
458 
459  if (Error E = VDataPtrOrErr.takeError())
460  return E;
461 
462  // Note that besides deserialization, this also performs the conversion for
463  // indirect call targets. The function pointers from the raw profile are
464  // remapped into function name hashes.
465  VDataPtrOrErr.get()->deserializeTo(Record, Symtab.get());
466  CurValueDataSize = VDataPtrOrErr.get()->getSize();
467  return success();
468 }
469 
470 template <class IntPtrT>
472  if (atEnd())
473  // At this point, ValueDataStart field points to the next header.
474  if (Error E = readNextHeader(getNextHeaderPos()))
475  return error(std::move(E));
476 
477  // Read name ad set it in Record.
478  if (Error E = readName(Record))
479  return error(std::move(E));
480 
481  // Read FuncHash and set it in Record.
482  if (Error E = readFuncHash(Record))
483  return error(std::move(E));
484 
485  // Read raw counts and set Record.
486  if (Error E = readRawCounts(Record))
487  return error(std::move(E));
488 
489  // Read value data and set Record.
490  if (Error E = readValueProfilingData(Record))
491  return error(std::move(E));
492 
493  // Iterate.
494  advanceData();
495  return success();
496 }
497 
498 namespace llvm {
499 
500 template class RawInstrProfReader<uint32_t>;
501 template class RawInstrProfReader<uint64_t>;
502 
503 } // end namespace llvm
504 
508 }
509 
512 
514  const unsigned char *&D, const unsigned char *const End) {
516  ValueProfData::getValueProfData(D, End, ValueProfDataEndianness);
517 
518  if (VDataPtrOrErr.takeError())
519  return false;
520 
521  VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr);
522  D += VDataPtrOrErr.get()->TotalSize;
523 
524  return true;
525 }
526 
528  offset_type N) {
529  using namespace support;
530 
531  // Check if the data is corrupt. If so, don't try to read it.
532  if (N % sizeof(uint64_t))
533  return data_type();
534 
535  DataBuffer.clear();
536  std::vector<uint64_t> CounterBuffer;
537 
538  const unsigned char *End = D + N;
539  while (D < End) {
540  // Read hash.
541  if (D + sizeof(uint64_t) >= End)
542  return data_type();
543  uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
544 
545  // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
546  uint64_t CountsSize = N / sizeof(uint64_t) - 1;
547  // If format version is different then read the number of counters.
548  if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
549  if (D + sizeof(uint64_t) > End)
550  return data_type();
551  CountsSize = endian::readNext<uint64_t, little, unaligned>(D);
552  }
553  // Read counter values.
554  if (D + CountsSize * sizeof(uint64_t) > End)
555  return data_type();
556 
557  CounterBuffer.clear();
558  CounterBuffer.reserve(CountsSize);
559  for (uint64_t J = 0; J < CountsSize; ++J)
560  CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
561 
562  DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
563 
564  // Read value profiling data.
565  if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
566  !readValueProfilingData(D, End)) {
567  DataBuffer.clear();
568  return data_type();
569  }
570  }
571  return DataBuffer;
572 }
573 
574 template <typename HashTableImpl>
577  auto Iter = HashTable->find(FuncName);
578  if (Iter == HashTable->end())
579  return make_error<InstrProfError>(instrprof_error::unknown_function);
580 
581  Data = (*Iter);
582  if (Data.empty())
583  return make_error<InstrProfError>(instrprof_error::malformed);
584 
585  return Error::success();
586 }
587 
588 template <typename HashTableImpl>
591  if (atEnd())
592  return make_error<InstrProfError>(instrprof_error::eof);
593 
594  Data = *RecordIterator;
595 
596  if (Data.empty())
597  return make_error<InstrProfError>(instrprof_error::malformed);
598 
599  return Error::success();
600 }
601 
602 template <typename HashTableImpl>
604  const unsigned char *Buckets, const unsigned char *const Payload,
605  const unsigned char *const Base, IndexedInstrProf::HashT HashType,
606  uint64_t Version) {
607  FormatVersion = Version;
608  HashTable.reset(HashTableImpl::Create(
609  Buckets, Payload, Base,
610  typename HashTableImpl::InfoType(HashType, Version)));
611  RecordIterator = HashTable->data_begin();
612 }
613 
614 namespace {
615 /// A remapper that does not apply any remappings.
616 class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
618 
619 public:
620  InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)
621  : Underlying(Underlying) {}
622 
623  Error getRecords(StringRef FuncName,
625  return Underlying.getRecords(FuncName, Data);
626  }
627 };
628 }
629 
630 /// A remapper that applies remappings based on a symbol remapping file.
631 template <typename HashTableImpl>
633  : public InstrProfReaderRemapper {
634 public:
636  std::unique_ptr<MemoryBuffer> RemapBuffer,
638  : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
639  }
640 
641  /// Extract the original function name from a PGO function name.
643  // We can have multiple :-separated pieces; there can be pieces both
644  // before and after the mangled name. Find the first part that starts
645  // with '_Z'; we'll assume that's the mangled name we want.
646  std::pair<StringRef, StringRef> Parts = {StringRef(), Name};
647  while (true) {
648  Parts = Parts.second.split(':');
649  if (Parts.first.startswith("_Z"))
650  return Parts.first;
651  if (Parts.second.empty())
652  return Name;
653  }
654  }
655 
656  /// Given a mangled name extracted from a PGO function name, and a new
657  /// form for that mangled name, reconstitute the name.
658  static void reconstituteName(StringRef OrigName, StringRef ExtractedName,
659  StringRef Replacement,
660  SmallVectorImpl<char> &Out) {
661  Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());
662  Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());
663  Out.insert(Out.end(), Replacement.begin(), Replacement.end());
664  Out.insert(Out.end(), ExtractedName.end(), OrigName.end());
665  }
666 
668  if (Error E = Remappings.read(*RemapBuffer))
669  return E;
670  for (StringRef Name : Underlying.HashTable->keys()) {
671  StringRef RealName = extractName(Name);
672  if (auto Key = Remappings.insert(RealName)) {
673  // FIXME: We could theoretically map the same equivalence class to
674  // multiple names in the profile data. If that happens, we should
675  // return NamedInstrProfRecords from all of them.
676  MappedNames.insert({Key, RealName});
677  }
678  }
679  return Error::success();
680  }
681 
684  StringRef RealName = extractName(FuncName);
685  if (auto Key = Remappings.lookup(RealName)) {
686  StringRef Remapped = MappedNames.lookup(Key);
687  if (!Remapped.empty()) {
688  if (RealName.begin() == FuncName.begin() &&
689  RealName.end() == FuncName.end())
690  FuncName = Remapped;
691  else {
692  // Try rebuilding the name from the given remapping.
693  SmallString<256> Reconstituted;
694  reconstituteName(FuncName, RealName, Remapped, Reconstituted);
695  Error E = Underlying.getRecords(Reconstituted, Data);
696  if (!E)
697  return E;
698 
699  // If we failed because the name doesn't exist, fall back to asking
700  // about the original name.
701  if (Error Unhandled = handleErrors(
702  std::move(E), [](std::unique_ptr<InstrProfError> Err) {
703  return Err->get() == instrprof_error::unknown_function
704  ? Error::success()
705  : Error(std::move(Err));
706  }))
707  return Unhandled;
708  }
709  }
710  }
711  return Underlying.getRecords(FuncName, Data);
712  }
713 
714 private:
715  /// The memory buffer containing the remapping configuration. Remappings
716  /// holds pointers into this buffer.
717  std::unique_ptr<MemoryBuffer> RemapBuffer;
718 
719  /// The mangling remapper.
720  SymbolRemappingReader Remappings;
721 
722  /// Mapping from mangled name keys to the name used for the key in the
723  /// profile data.
724  /// FIXME: Can we store a location within the on-disk hash table instead of
725  /// redoing lookup?
727 
728  /// The real profile data reader.
730 };
731 
733  using namespace support;
734 
735  if (DataBuffer.getBufferSize() < 8)
736  return false;
737  uint64_t Magic =
738  endian::read<uint64_t, little, aligned>(DataBuffer.getBufferStart());
739  // Verify that it's magical.
740  return Magic == IndexedInstrProf::Magic;
741 }
742 
743 const unsigned char *
744 IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
745  const unsigned char *Cur, bool UseCS) {
746  using namespace IndexedInstrProf;
747  using namespace support;
748 
750  const IndexedInstrProf::Summary *SummaryInLE =
751  reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
752  uint64_t NFields =
753  endian::byte_swap<uint64_t, little>(SummaryInLE->NumSummaryFields);
754  uint64_t NEntries =
755  endian::byte_swap<uint64_t, little>(SummaryInLE->NumCutoffEntries);
756  uint32_t SummarySize =
757  IndexedInstrProf::Summary::getSize(NFields, NEntries);
758  std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
759  IndexedInstrProf::allocSummary(SummarySize);
760 
761  const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
762  uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
763  for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
764  Dst[I] = endian::byte_swap<uint64_t, little>(Src[I]);
765 
766  SummaryEntryVector DetailedSummary;
767  for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
768  const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);
769  DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,
770  Ent.NumBlocks);
771  }
772  std::unique_ptr<llvm::ProfileSummary> &Summary =
773  UseCS ? this->CS_Summary : this->Summary;
774 
775  // initialize InstrProfSummary using the SummaryData from disk.
776  Summary = std::make_unique<ProfileSummary>(
778  DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
779  SummaryData->get(Summary::MaxBlockCount),
780  SummaryData->get(Summary::MaxInternalBlockCount),
781  SummaryData->get(Summary::MaxFunctionCount),
782  SummaryData->get(Summary::TotalNumBlocks),
783  SummaryData->get(Summary::TotalNumFunctions));
784  return Cur + SummarySize;
785  } else {
786  // The older versions do not support a profile summary. This just computes
787  // an empty summary, which will not result in accurate hot/cold detection.
788  // We would need to call addRecord for all NamedInstrProfRecords to get the
789  // correct summary. However, this version is old (prior to early 2016) and
790  // has not been supporting an accurate summary for several years.
792  Summary = Builder.getSummary();
793  return Cur;
794  }
795 }
796 
798  using namespace support;
799 
800  const unsigned char *Start =
801  (const unsigned char *)DataBuffer->getBufferStart();
802  const unsigned char *Cur = Start;
803  if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
805 
806  auto *Header = reinterpret_cast<const IndexedInstrProf::Header *>(Cur);
807  Cur += sizeof(IndexedInstrProf::Header);
808 
809  // Check the magic number.
810  uint64_t Magic = endian::byte_swap<uint64_t, little>(Header->Magic);
811  if (Magic != IndexedInstrProf::Magic)
813 
814  // Read the version.
815  uint64_t FormatVersion = endian::byte_swap<uint64_t, little>(Header->Version);
816  if (GET_VERSION(FormatVersion) >
819 
820  Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur,
821  /* UseCS */ false);
822  if (FormatVersion & VARIANT_MASK_CSIR_PROF)
823  Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur,
824  /* UseCS */ true);
825 
826  // Read the hash type and start offset.
828  endian::byte_swap<uint64_t, little>(Header->HashType));
829  if (HashType > IndexedInstrProf::HashT::Last)
831 
832  uint64_t HashOffset = endian::byte_swap<uint64_t, little>(Header->HashOffset);
833 
834  // The rest of the file is an on disk hash table.
835  auto IndexPtr =
836  std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
837  Start + HashOffset, Cur, Start, HashType, FormatVersion);
838 
839  // Load the remapping table now if requested.
840  if (RemappingBuffer) {
841  Remapper = std::make_unique<
843  std::move(RemappingBuffer), *IndexPtr);
844  if (Error E = Remapper->populateRemappings())
845  return E;
846  } else {
847  Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
848  }
849  Index = std::move(IndexPtr);
850 
851  return success();
852 }
853 
855  if (Symtab.get())
856  return *Symtab.get();
857 
858  std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
859  if (Error E = Index->populateSymtab(*NewSymtab.get())) {
861  }
862 
863  Symtab = std::move(NewSymtab);
864  return *Symtab.get();
865 }
866 
869  uint64_t FuncHash) {
871  Error Err = Remapper->getRecords(FuncName, Data);
872  if (Err)
873  return std::move(Err);
874  // Found it. Look for counters with the right hash.
875  for (unsigned I = 0, E = Data.size(); I < E; ++I) {
876  // Check for a match and fill the vector if there is one.
877  if (Data[I].Hash == FuncHash) {
878  return std::move(Data[I]);
879  }
880  }
882 }
883 
885  uint64_t FuncHash,
886  std::vector<uint64_t> &Counts) {
887  Expected<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash);
888  if (Error E = Record.takeError())
889  return error(std::move(E));
890 
891  Counts = Record.get().Counts;
892  return success();
893 }
894 
897 
898  Error E = Index->getRecords(Data);
899  if (E)
900  return error(std::move(E));
901 
902  Record = Data[RecordIndex++];
903  if (RecordIndex >= Data.size()) {
904  Index->advanceToNextKey();
905  RecordIndex = 0;
906  }
907  return success();
908 }
909 
911  uint64_t NumFuncs = 0;
912  for (const auto &Func : *this) {
913  if (isIRLevelProfile()) {
914  bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash);
915  if (FuncIsCS != IsCS)
916  continue;
917  }
918  Func.accumulateCounts(Sum);
919  ++NumFuncs;
920  }
921  Sum.NumEntries = NumFuncs;
922 }
#define VP_READ_ADVANCE(Val)
Represents either an error or a value T.
Definition: ErrorOr.h:56
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:406
InfoType
FunctionInfo information type that is used to encode the optional data that is associated with a Func...
Error readHeader() override
Read the header. Required before reading first record.
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1037
bool isPrint(char C)
Checks whether character C is printable.
Definition: StringExtras.h:105
This class represents lattice values for constants.
Definition: AllocatorList.h:23
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1003
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Definition: StringRef.h:181
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:707
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
static instrprof_error take(Error E)
Consume an Error and return the raw enum value contained within it.
Definition: InstrProf.h:319
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:503
A remapper that applies remappings based on a symbol remapping file.
Error error(instrprof_error Err)
Set the current error and return same.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1077
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:64
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1165
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
size_t getBufferSize() const
Definition: MemoryBuffer.h:61
Error takeError()
Take ownership of the stored error.
Definition: Error.h:552
void reserve(size_type N)
Definition: SmallVector.h:369
InstrProfLookupTrait::offset_type offset_type
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
Definition: InstrProf.h:916
Error readHeader() override
Read the header.
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
const uint64_t Magic
Definition: InstrProf.h:956
Definition: BitVector.h:937
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path)
Factory method to create an appropriately typed reader for the given instrprof file.
const HashT HashType
Definition: InstrProf.h:982
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
static StringRef getName(Value *V)
Tagged union holding either a T or a Error.
Definition: yaml2obj.h:21
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:140
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name&#39;s MD5 hash.
Definition: InstrProf.h:481
Key
PAL metadata keys.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:144
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
const uint64_t Version
Definition: InstrProf.h:1095
Reader for the simple text based instrprof format.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1004
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
static bool hasFormat(const MemoryBuffer &DataBuffer)
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1231
InstrProfLookupTrait::data_type data_type
hash_value_type ComputeHash(StringRef K)
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
ArrayRef< NamedInstrProfRecord > data_type
Base class and interface for reading profiling data of any known instrprof format.
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1001
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:87
ValueKind
Value kinds.
std::error_code getError() const
Definition: ErrorOr.h:159
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
std::unique_ptr< InstrProfSymtab > Symtab
std::underlying_type< E >::type Underlying(E Val)
Check that Val is in range for E, and return Val cast to E&#39;s underlying type.
Definition: BitmaskEnum.h:90
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:854
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:984
virtual Error getRecords(ArrayRef< NamedInstrProfRecord > &Data)=0
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1001
#define CHECK_LINE_END(Line)
#define READ_NUM(Str, Dst)
void clearValueData()
Clear value data entries.
Definition: InstrProf.h:768
static const char *const Magic
Definition: Archive.cpp:41
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Path)
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Name matcher supporting fuzzy matching of symbol names to names in profiles.
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:390
RawInstrProfReader< uint32_t > RawInstrProfReader32
std::unique_ptr< ProfileSummary > getSummary()
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:710
A file format agnostic iterator over profiling data.
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
Reader for the raw instrprof binary format from runtime.
reference get()
Returns a reference to the stored T value.
Definition: Error.h:532
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
RawInstrProfReader< uint64_t > RawInstrProfReader64
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:41
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:467
iterator begin() const
Definition: StringRef.h:115
unsigned char getSwappedBytes(unsigned char C)
Definition: SwapByteOrder.h:98
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, int64_t FileSize=-1, bool RequiresNullTerminator=true)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
std::vector< uint64_t > Counts
Definition: InstrProf.h:686
Error readHeader() override
Read the file header.
virtual Error readHeader()=0
Read the header. Required before reading first record.
Profiling information for a single function.
Definition: InstrProf.h:685
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Reader for symbol remapping files.
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name...
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
const char * getBufferStart() const
Definition: MemoryBuffer.h:59
virtual bool isIRLevelProfile() const =0
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
Provides ErrorOr<T> smart pointer.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:901
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
Reader for the indexed binary instrprof format.
std::vector< ProfileSummaryEntry > SummaryEntryVector
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, const Twine &RemappingPath="")
Factory method to create an indexed reader.
iterator end() const
Definition: StringRef.h:117
const uint64_t Version
Definition: InstrProf.h:980
void Clear()
Clear value data entries and edge counters.
Definition: InstrProf.h:762
reference get()
Definition: ErrorOr.h:156
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:143
Error success()
Clear the current error and return a successful one.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
static Error initializeReader(InstrProfReader &Reader)