LLVM  10.0.0svn
SampleProfReader.cpp
Go to the documentation of this file.
1 //===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class that reads LLVM sample profiles. It
10 // supports three file formats: text, binary and gcov.
11 //
12 // The textual representation is useful for debugging and testing purposes. The
13 // binary representation is more compact, resulting in smaller file sizes.
14 //
15 // The gcov encoding is the one generated by GCC's AutoFDO profile creation
16 // tool (https://github.com/google/autofdo)
17 //
18 // All three encodings can be used interchangeably as an input sample profile.
19 //
20 //===----------------------------------------------------------------------===//
21 
23 #include "llvm/ADT/DenseMap.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/IR/ProfileSummary.h"
29 #include "llvm/Support/ErrorOr.h"
30 #include "llvm/Support/LEB128.h"
32 #include "llvm/Support/MD5.h"
35 #include <algorithm>
36 #include <cstddef>
37 #include <cstdint>
38 #include <limits>
39 #include <memory>
40 #include <system_error>
41 #include <vector>
42 
43 using namespace llvm;
44 using namespace sampleprof;
45 
46 /// Dump the function profile for \p FName.
47 ///
48 /// \param FName Name of the function to print.
49 /// \param OS Stream to emit the output to.
51  raw_ostream &OS) {
52  OS << "Function: " << FName << ": " << Profiles[FName];
53 }
54 
55 /// Dump all the function profiles found on stream \p OS.
57  for (const auto &I : Profiles)
58  dumpFunctionProfile(I.getKey(), OS);
59 }
60 
61 /// Parse \p Input as function head.
62 ///
63 /// Parse one line of \p Input, and update function name in \p FName,
64 /// function's total sample count in \p NumSamples, function's entry
65 /// count in \p NumHeadSamples.
66 ///
67 /// \returns true if parsing is successful.
68 static bool ParseHead(const StringRef &Input, StringRef &FName,
69  uint64_t &NumSamples, uint64_t &NumHeadSamples) {
70  if (Input[0] == ' ')
71  return false;
72  size_t n2 = Input.rfind(':');
73  size_t n1 = Input.rfind(':', n2 - 1);
74  FName = Input.substr(0, n1);
75  if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples))
76  return false;
77  if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples))
78  return false;
79  return true;
80 }
81 
82 /// Returns true if line offset \p L is legal (only has 16 bits).
83 static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
84 
85 /// Parse \p Input as line sample.
86 ///
87 /// \param Input input line.
88 /// \param IsCallsite true if the line represents an inlined callsite.
89 /// \param Depth the depth of the inline stack.
90 /// \param NumSamples total samples of the line/inlined callsite.
91 /// \param LineOffset line offset to the start of the function.
92 /// \param Discriminator discriminator of the line.
93 /// \param TargetCountMap map from indirect call target to count.
94 ///
95 /// returns true if parsing is successful.
96 static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth,
97  uint64_t &NumSamples, uint32_t &LineOffset,
98  uint32_t &Discriminator, StringRef &CalleeName,
99  DenseMap<StringRef, uint64_t> &TargetCountMap) {
100  for (Depth = 0; Input[Depth] == ' '; Depth++)
101  ;
102  if (Depth == 0)
103  return false;
104 
105  size_t n1 = Input.find(':');
106  StringRef Loc = Input.substr(Depth, n1 - Depth);
107  size_t n2 = Loc.find('.');
108  if (n2 == StringRef::npos) {
109  if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset))
110  return false;
111  Discriminator = 0;
112  } else {
113  if (Loc.substr(0, n2).getAsInteger(10, LineOffset))
114  return false;
115  if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator))
116  return false;
117  }
118 
119  StringRef Rest = Input.substr(n1 + 2);
120  if (Rest[0] >= '0' && Rest[0] <= '9') {
121  IsCallsite = false;
122  size_t n3 = Rest.find(' ');
123  if (n3 == StringRef::npos) {
124  if (Rest.getAsInteger(10, NumSamples))
125  return false;
126  } else {
127  if (Rest.substr(0, n3).getAsInteger(10, NumSamples))
128  return false;
129  }
130  // Find call targets and their sample counts.
131  // Note: In some cases, there are symbols in the profile which are not
132  // mangled. To accommodate such cases, use colon + integer pairs as the
133  // anchor points.
134  // An example:
135  // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
136  // ":1000" and ":437" are used as anchor points so the string above will
137  // be interpreted as
138  // target: _M_construct<char *>
139  // count: 1000
140  // target: string_view<std::allocator<char> >
141  // count: 437
142  while (n3 != StringRef::npos) {
143  n3 += Rest.substr(n3).find_first_not_of(' ');
144  Rest = Rest.substr(n3);
145  n3 = Rest.find_first_of(':');
146  if (n3 == StringRef::npos || n3 == 0)
147  return false;
148 
150  uint64_t count, n4;
151  while (true) {
152  // Get the segment after the current colon.
153  StringRef AfterColon = Rest.substr(n3 + 1);
154  // Get the target symbol before the current colon.
155  Target = Rest.substr(0, n3);
156  // Check if the word after the current colon is an integer.
157  n4 = AfterColon.find_first_of(' ');
158  n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
159  StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1);
160  if (!WordAfterColon.getAsInteger(10, count))
161  break;
162 
163  // Try to find the next colon.
164  uint64_t n5 = AfterColon.find_first_of(':');
165  if (n5 == StringRef::npos)
166  return false;
167  n3 += n5 + 1;
168  }
169 
170  // An anchor point is found. Save the {target, count} pair
171  TargetCountMap[Target] = count;
172  if (n4 == Rest.size())
173  break;
174  // Change n3 to the next blank space after colon + integer pair.
175  n3 = n4;
176  }
177  } else {
178  IsCallsite = true;
179  size_t n3 = Rest.find_last_of(':');
180  CalleeName = Rest.substr(0, n3);
181  if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples))
182  return false;
183  }
184  return true;
185 }
186 
187 /// Load samples from a text file.
188 ///
189 /// See the documentation at the top of the file for an explanation of
190 /// the expected format.
191 ///
192 /// \returns true if the file was loaded successfully, false otherwise.
193 std::error_code SampleProfileReaderText::read() {
194  line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
196 
197  InlineCallStack InlineStack;
198 
199  for (; !LineIt.is_at_eof(); ++LineIt) {
200  if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#')
201  continue;
202  // Read the header of each function.
203  //
204  // Note that for function identifiers we are actually expecting
205  // mangled names, but we may not always get them. This happens when
206  // the compiler decides not to emit the function (e.g., it was inlined
207  // and removed). In this case, the binary will not have the linkage
208  // name for the function, so the profiler will emit the function's
209  // unmangled name, which may contain characters like ':' and '>' in its
210  // name (member functions, templates, etc).
211  //
212  // The only requirement we place on the identifier, then, is that it
213  // should not begin with a number.
214  if ((*LineIt)[0] != ' ') {
215  uint64_t NumSamples, NumHeadSamples;
216  StringRef FName;
217  if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) {
218  reportError(LineIt.line_number(),
219  "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
221  }
222  Profiles[FName] = FunctionSamples();
223  FunctionSamples &FProfile = Profiles[FName];
224  FProfile.setName(FName);
225  MergeResult(Result, FProfile.addTotalSamples(NumSamples));
226  MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
227  InlineStack.clear();
228  InlineStack.push_back(&FProfile);
229  } else {
230  uint64_t NumSamples;
231  StringRef FName;
232  DenseMap<StringRef, uint64_t> TargetCountMap;
233  bool IsCallsite;
234  uint32_t Depth, LineOffset, Discriminator;
235  if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset,
236  Discriminator, FName, TargetCountMap)) {
237  reportError(LineIt.line_number(),
238  "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
239  *LineIt);
241  }
242  if (IsCallsite) {
243  while (InlineStack.size() > Depth) {
244  InlineStack.pop_back();
245  }
246  FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
247  LineLocation(LineOffset, Discriminator))[FName];
248  FSamples.setName(FName);
249  MergeResult(Result, FSamples.addTotalSamples(NumSamples));
250  InlineStack.push_back(&FSamples);
251  } else {
252  while (InlineStack.size() > Depth) {
253  InlineStack.pop_back();
254  }
255  FunctionSamples &FProfile = *InlineStack.back();
256  for (const auto &name_count : TargetCountMap) {
257  MergeResult(Result, FProfile.addCalledTargetSamples(
258  LineOffset, Discriminator, name_count.first,
259  name_count.second));
260  }
261  MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
262  NumSamples));
263  }
264  }
265  }
266  if (Result == sampleprof_error::success)
267  computeSummary();
268 
269  return Result;
270 }
271 
273  bool result = false;
274 
275  // Check that the first non-comment line is a valid function header.
276  line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
277  if (!LineIt.is_at_eof()) {
278  if ((*LineIt)[0] != ' ') {
279  uint64_t NumSamples, NumHeadSamples;
280  StringRef FName;
281  result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples);
282  }
283  }
284 
285  return result;
286 }
287 
289  unsigned NumBytesRead = 0;
290  std::error_code EC;
291  uint64_t Val = decodeULEB128(Data, &NumBytesRead);
292 
293  if (Val > std::numeric_limits<T>::max())
295  else if (Data + NumBytesRead > End)
297  else
299 
300  if (EC) {
301  reportError(0, EC.message());
302  return EC;
303  }
304 
305  Data += NumBytesRead;
306  return static_cast<T>(Val);
307 }
308 
310  std::error_code EC;
311  StringRef Str(reinterpret_cast<const char *>(Data));
312  if (Data + Str.size() + 1 > End) {
314  reportError(0, EC.message());
315  return EC;
316  }
317 
318  Data += Str.size() + 1;
319  return Str;
320 }
321 
322 template <typename T>
324  std::error_code EC;
325 
326  if (Data + sizeof(T) > End) {
328  reportError(0, EC.message());
329  return EC;
330  }
331 
332  using namespace support;
333  T Val = endian::readNext<T, little, unaligned>(Data);
334  return Val;
335 }
336 
337 template <typename T>
339  std::error_code EC;
340  auto Idx = readNumber<uint32_t>();
341  if (std::error_code EC = Idx.getError())
342  return EC;
343  if (*Idx >= Table.size())
345  return *Idx;
346 }
347 
349  auto Idx = readStringIndex(NameTable);
350  if (std::error_code EC = Idx.getError())
351  return EC;
352 
353  return NameTable[*Idx];
354 }
355 
356 ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() {
357  auto Idx = readStringIndex(NameTable);
358  if (std::error_code EC = Idx.getError())
359  return EC;
360 
361  return StringRef(NameTable[*Idx]);
362 }
363 
364 std::error_code
366  auto NumSamples = readNumber<uint64_t>();
367  if (std::error_code EC = NumSamples.getError())
368  return EC;
369  FProfile.addTotalSamples(*NumSamples);
370 
371  // Read the samples in the body.
372  auto NumRecords = readNumber<uint32_t>();
373  if (std::error_code EC = NumRecords.getError())
374  return EC;
375 
376  for (uint32_t I = 0; I < *NumRecords; ++I) {
377  auto LineOffset = readNumber<uint64_t>();
378  if (std::error_code EC = LineOffset.getError())
379  return EC;
380 
381  if (!isOffsetLegal(*LineOffset)) {
382  return std::error_code();
383  }
384 
385  auto Discriminator = readNumber<uint64_t>();
386  if (std::error_code EC = Discriminator.getError())
387  return EC;
388 
389  auto NumSamples = readNumber<uint64_t>();
390  if (std::error_code EC = NumSamples.getError())
391  return EC;
392 
393  auto NumCalls = readNumber<uint32_t>();
394  if (std::error_code EC = NumCalls.getError())
395  return EC;
396 
397  for (uint32_t J = 0; J < *NumCalls; ++J) {
398  auto CalledFunction(readStringFromTable());
399  if (std::error_code EC = CalledFunction.getError())
400  return EC;
401 
402  auto CalledFunctionSamples = readNumber<uint64_t>();
403  if (std::error_code EC = CalledFunctionSamples.getError())
404  return EC;
405 
406  FProfile.addCalledTargetSamples(*LineOffset, *Discriminator,
407  *CalledFunction, *CalledFunctionSamples);
408  }
409 
410  FProfile.addBodySamples(*LineOffset, *Discriminator, *NumSamples);
411  }
412 
413  // Read all the samples for inlined function calls.
414  auto NumCallsites = readNumber<uint32_t>();
415  if (std::error_code EC = NumCallsites.getError())
416  return EC;
417 
418  for (uint32_t J = 0; J < *NumCallsites; ++J) {
419  auto LineOffset = readNumber<uint64_t>();
420  if (std::error_code EC = LineOffset.getError())
421  return EC;
422 
423  auto Discriminator = readNumber<uint64_t>();
424  if (std::error_code EC = Discriminator.getError())
425  return EC;
426 
427  auto FName(readStringFromTable());
428  if (std::error_code EC = FName.getError())
429  return EC;
430 
431  FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
432  LineLocation(*LineOffset, *Discriminator))[*FName];
433  CalleeProfile.setName(*FName);
434  if (std::error_code EC = readProfile(CalleeProfile))
435  return EC;
436  }
437 
439 }
440 
442  auto NumHeadSamples = readNumber<uint64_t>();
443  if (std::error_code EC = NumHeadSamples.getError())
444  return EC;
445 
446  auto FName(readStringFromTable());
447  if (std::error_code EC = FName.getError())
448  return EC;
449 
450  Profiles[*FName] = FunctionSamples();
451  FunctionSamples &FProfile = Profiles[*FName];
452  FProfile.setName(*FName);
453 
454  FProfile.addHeadSamples(*NumHeadSamples);
455 
456  if (std::error_code EC = readProfile(FProfile))
457  return EC;
459 }
460 
462  while (!at_eof()) {
463  if (std::error_code EC = readFuncProfile())
464  return EC;
465  }
466 
468 }
469 
471  const uint8_t *BufStart =
472  reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
473 
474  for (auto &Entry : SecHdrTable) {
475  // Skip empty section.
476  if (!Entry.Size)
477  continue;
478  Data = BufStart + Entry.Offset;
479  switch (Entry.Type) {
480  case SecProfSummary:
481  if (std::error_code EC = readSummary())
482  return EC;
483  break;
484  case SecNameTable:
485  if (std::error_code EC = readNameTable())
486  return EC;
487  break;
488  case SecLBRProfile:
489  while (Data < BufStart + Entry.Offset + Entry.Size) {
490  if (std::error_code EC = readFuncProfile())
491  return EC;
492  }
493  break;
494  default:
495  continue;
496  }
497  if (Data != BufStart + Entry.Offset + Entry.Size)
499  }
500 
502 }
503 
505  std::vector<uint64_t> OffsetsToUse;
506  if (UseAllFuncs) {
507  for (auto FuncEntry : FuncOffsetTable) {
508  OffsetsToUse.push_back(FuncEntry.second);
509  }
510  }
511  else {
512  for (auto Name : FuncsToUse) {
513  auto GUID = std::to_string(MD5Hash(Name));
514  auto iter = FuncOffsetTable.find(StringRef(GUID));
515  if (iter == FuncOffsetTable.end())
516  continue;
517  OffsetsToUse.push_back(iter->second);
518  }
519  }
520 
521  for (auto Offset : OffsetsToUse) {
522  const uint8_t *SavedData = Data;
523  Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
524  Offset;
525  if (std::error_code EC = readFuncProfile())
526  return EC;
527  Data = SavedData;
528  }
530 }
531 
532 std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
533  if (Magic == SPMagic())
536 }
537 
538 std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
539  if (Magic == SPMagic(SPF_Ext_Binary))
542 }
543 
544 std::error_code
545 SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) {
546  if (Magic == SPMagic(SPF_Compact_Binary))
549 }
550 
552  auto Size = readNumber<uint32_t>();
553  if (std::error_code EC = Size.getError())
554  return EC;
555  NameTable.reserve(*Size);
556  for (uint32_t I = 0; I < *Size; ++I) {
557  auto Name(readString());
558  if (std::error_code EC = Name.getError())
559  return EC;
560  NameTable.push_back(*Name);
561  }
562 
564 }
565 
566 std::error_code SampleProfileReaderCompactBinary::readNameTable() {
567  auto Size = readNumber<uint64_t>();
568  if (std::error_code EC = Size.getError())
569  return EC;
570  NameTable.reserve(*Size);
571  for (uint32_t I = 0; I < *Size; ++I) {
572  auto FID = readNumber<uint64_t>();
573  if (std::error_code EC = FID.getError())
574  return EC;
575  NameTable.push_back(std::to_string(*FID));
576  }
578 }
579 
582  auto Type = readUnencodedNumber<uint64_t>();
583  if (std::error_code EC = Type.getError())
584  return EC;
585  Entry.Type = static_cast<SecType>(*Type);
586 
587  auto Flag = readUnencodedNumber<uint64_t>();
588  if (std::error_code EC = Flag.getError())
589  return EC;
590  Entry.Flag = *Flag;
591 
592  auto Offset = readUnencodedNumber<uint64_t>();
593  if (std::error_code EC = Offset.getError())
594  return EC;
595  Entry.Offset = *Offset;
596 
597  auto Size = readUnencodedNumber<uint64_t>();
598  if (std::error_code EC = Size.getError())
599  return EC;
600  Entry.Size = *Size;
601 
602  SecHdrTable.push_back(std::move(Entry));
604 }
605 
607  auto EntryNum = readUnencodedNumber<uint64_t>();
608  if (std::error_code EC = EntryNum.getError())
609  return EC;
610 
611  for (uint32_t i = 0; i < (*EntryNum); i++)
612  if (std::error_code EC = readSecHdrTableEntry())
613  return EC;
614 
616 }
617 
619  const uint8_t *BufStart =
620  reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
621  Data = BufStart;
622  End = BufStart + Buffer->getBufferSize();
623 
624  if (std::error_code EC = readMagicIdent())
625  return EC;
626 
627  if (std::error_code EC = readSecHdrTable())
628  return EC;
629 
631 }
632 
634  // Read and check the magic identifier.
635  auto Magic = readNumber<uint64_t>();
636  if (std::error_code EC = Magic.getError())
637  return EC;
638  else if (std::error_code EC = verifySPMagic(*Magic))
639  return EC;
640 
641  // Read the version number.
642  auto Version = readNumber<uint64_t>();
643  if (std::error_code EC = Version.getError())
644  return EC;
645  else if (*Version != SPVersion())
647 
649 }
650 
652  Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
653  End = Data + Buffer->getBufferSize();
654 
655  if (std::error_code EC = readMagicIdent())
656  return EC;
657 
658  if (std::error_code EC = readSummary())
659  return EC;
660 
661  if (std::error_code EC = readNameTable())
662  return EC;
664 }
665 
666 std::error_code SampleProfileReaderCompactBinary::readHeader() {
668  if (std::error_code EC = readFuncOffsetTable())
669  return EC;
671 }
672 
673 std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() {
674  auto TableOffset = readUnencodedNumber<uint64_t>();
675  if (std::error_code EC = TableOffset.getError())
676  return EC;
677 
678  const uint8_t *SavedData = Data;
679  const uint8_t *TableStart =
680  reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
681  *TableOffset;
682  Data = TableStart;
683 
684  auto Size = readNumber<uint64_t>();
685  if (std::error_code EC = Size.getError())
686  return EC;
687 
688  FuncOffsetTable.reserve(*Size);
689  for (uint32_t I = 0; I < *Size; ++I) {
690  auto FName(readStringFromTable());
691  if (std::error_code EC = FName.getError())
692  return EC;
693 
694  auto Offset = readNumber<uint64_t>();
695  if (std::error_code EC = Offset.getError())
696  return EC;
697 
698  FuncOffsetTable[*FName] = *Offset;
699  }
700  End = TableStart;
701  Data = SavedData;
703 }
704 
706  UseAllFuncs = false;
707  FuncsToUse.clear();
708  for (auto &F : M) {
710  FuncsToUse.insert(CanonName);
711  }
712 }
713 
714 std::error_code SampleProfileReaderBinary::readSummaryEntry(
715  std::vector<ProfileSummaryEntry> &Entries) {
716  auto Cutoff = readNumber<uint64_t>();
717  if (std::error_code EC = Cutoff.getError())
718  return EC;
719 
720  auto MinBlockCount = readNumber<uint64_t>();
721  if (std::error_code EC = MinBlockCount.getError())
722  return EC;
723 
724  auto NumBlocks = readNumber<uint64_t>();
725  if (std::error_code EC = NumBlocks.getError())
726  return EC;
727 
728  Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
730 }
731 
733  auto TotalCount = readNumber<uint64_t>();
734  if (std::error_code EC = TotalCount.getError())
735  return EC;
736 
737  auto MaxBlockCount = readNumber<uint64_t>();
738  if (std::error_code EC = MaxBlockCount.getError())
739  return EC;
740 
741  auto MaxFunctionCount = readNumber<uint64_t>();
742  if (std::error_code EC = MaxFunctionCount.getError())
743  return EC;
744 
745  auto NumBlocks = readNumber<uint64_t>();
746  if (std::error_code EC = NumBlocks.getError())
747  return EC;
748 
749  auto NumFunctions = readNumber<uint64_t>();
750  if (std::error_code EC = NumFunctions.getError())
751  return EC;
752 
753  auto NumSummaryEntries = readNumber<uint64_t>();
754  if (std::error_code EC = NumSummaryEntries.getError())
755  return EC;
756 
757  std::vector<ProfileSummaryEntry> Entries;
758  for (unsigned i = 0; i < *NumSummaryEntries; i++) {
759  std::error_code EC = readSummaryEntry(Entries);
760  if (EC != sampleprof_error::success)
761  return EC;
762  }
763  Summary = std::make_unique<ProfileSummary>(
764  ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0,
765  *MaxFunctionCount, *NumBlocks, *NumFunctions);
766 
768 }
769 
771  const uint8_t *Data =
772  reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
773  uint64_t Magic = decodeULEB128(Data);
774  return Magic == SPMagic();
775 }
776 
778  const uint8_t *Data =
779  reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
780  uint64_t Magic = decodeULEB128(Data);
781  return Magic == SPMagic(SPF_Ext_Binary);
782 }
783 
785  const uint8_t *Data =
786  reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
787  uint64_t Magic = decodeULEB128(Data);
788  return Magic == SPMagic(SPF_Compact_Binary);
789 }
790 
792  uint32_t dummy;
793  if (!GcovBuffer.readInt(dummy))
796 }
797 
799  if (sizeof(T) <= sizeof(uint32_t)) {
800  uint32_t Val;
801  if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
802  return static_cast<T>(Val);
803  } else if (sizeof(T) <= sizeof(uint64_t)) {
804  uint64_t Val;
805  if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
806  return static_cast<T>(Val);
807  }
808 
809  std::error_code EC = sampleprof_error::malformed;
810  reportError(0, EC.message());
811  return EC;
812 }
813 
815  StringRef Str;
816  if (!GcovBuffer.readString(Str))
818  return Str;
819 }
820 
822  // Read the magic identifier.
823  if (!GcovBuffer.readGCDAFormat())
825 
826  // Read the version number. Note - the GCC reader does not validate this
827  // version, but the profile creator generates v704.
828  GCOV::GCOVVersion version;
829  if (!GcovBuffer.readGCOVVersion(version))
831 
832  if (version != GCOV::V704)
834 
835  // Skip the empty integer.
836  if (std::error_code EC = skipNextWord())
837  return EC;
838 
840 }
841 
843  uint32_t Tag;
844  if (!GcovBuffer.readInt(Tag))
846 
847  if (Tag != Expected)
849 
850  if (std::error_code EC = skipNextWord())
851  return EC;
852 
854 }
855 
857  if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames))
858  return EC;
859 
860  uint32_t Size;
861  if (!GcovBuffer.readInt(Size))
863 
864  for (uint32_t I = 0; I < Size; ++I) {
865  StringRef Str;
866  if (!GcovBuffer.readString(Str))
868  Names.push_back(Str);
869  }
870 
872 }
873 
875  if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction))
876  return EC;
877 
878  uint32_t NumFunctions;
879  if (!GcovBuffer.readInt(NumFunctions))
881 
882  InlineCallStack Stack;
883  for (uint32_t I = 0; I < NumFunctions; ++I)
884  if (std::error_code EC = readOneFunctionProfile(Stack, true, 0))
885  return EC;
886 
887  computeSummary();
889 }
890 
892  const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
893  uint64_t HeadCount = 0;
894  if (InlineStack.size() == 0)
895  if (!GcovBuffer.readInt64(HeadCount))
897 
898  uint32_t NameIdx;
899  if (!GcovBuffer.readInt(NameIdx))
901 
902  StringRef Name(Names[NameIdx]);
903 
904  uint32_t NumPosCounts;
905  if (!GcovBuffer.readInt(NumPosCounts))
907 
908  uint32_t NumCallsites;
909  if (!GcovBuffer.readInt(NumCallsites))
911 
912  FunctionSamples *FProfile = nullptr;
913  if (InlineStack.size() == 0) {
914  // If this is a top function that we have already processed, do not
915  // update its profile again. This happens in the presence of
916  // function aliases. Since these aliases share the same function
917  // body, there will be identical replicated profiles for the
918  // original function. In this case, we simply not bother updating
919  // the profile of the original function.
920  FProfile = &Profiles[Name];
921  FProfile->addHeadSamples(HeadCount);
922  if (FProfile->getTotalSamples() > 0)
923  Update = false;
924  } else {
925  // Otherwise, we are reading an inlined instance. The top of the
926  // inline stack contains the profile of the caller. Insert this
927  // callee in the caller's CallsiteMap.
928  FunctionSamples *CallerProfile = InlineStack.front();
929  uint32_t LineOffset = Offset >> 16;
930  uint32_t Discriminator = Offset & 0xffff;
931  FProfile = &CallerProfile->functionSamplesAt(
932  LineLocation(LineOffset, Discriminator))[Name];
933  }
934  FProfile->setName(Name);
935 
936  for (uint32_t I = 0; I < NumPosCounts; ++I) {
938  if (!GcovBuffer.readInt(Offset))
940 
941  uint32_t NumTargets;
942  if (!GcovBuffer.readInt(NumTargets))
944 
945  uint64_t Count;
946  if (!GcovBuffer.readInt64(Count))
948 
949  // The line location is encoded in the offset as:
950  // high 16 bits: line offset to the start of the function.
951  // low 16 bits: discriminator.
952  uint32_t LineOffset = Offset >> 16;
953  uint32_t Discriminator = Offset & 0xffff;
954 
955  InlineCallStack NewStack;
956  NewStack.push_back(FProfile);
957  NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end());
958  if (Update) {
959  // Walk up the inline stack, adding the samples on this line to
960  // the total sample count of the callers in the chain.
961  for (auto CallerProfile : NewStack)
962  CallerProfile->addTotalSamples(Count);
963 
964  // Update the body samples for the current profile.
965  FProfile->addBodySamples(LineOffset, Discriminator, Count);
966  }
967 
968  // Process the list of functions called at an indirect call site.
969  // These are all the targets that a function pointer (or virtual
970  // function) resolved at runtime.
971  for (uint32_t J = 0; J < NumTargets; J++) {
972  uint32_t HistVal;
973  if (!GcovBuffer.readInt(HistVal))
975 
976  if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
978 
979  uint64_t TargetIdx;
980  if (!GcovBuffer.readInt64(TargetIdx))
982  StringRef TargetName(Names[TargetIdx]);
983 
984  uint64_t TargetCount;
985  if (!GcovBuffer.readInt64(TargetCount))
987 
988  if (Update)
989  FProfile->addCalledTargetSamples(LineOffset, Discriminator,
990  TargetName, TargetCount);
991  }
992  }
993 
994  // Process all the inlined callers into the current function. These
995  // are all the callsites that were inlined into this function.
996  for (uint32_t I = 0; I < NumCallsites; I++) {
997  // The offset is encoded as:
998  // high 16 bits: line offset to the start of the function.
999  // low 16 bits: discriminator.
1000  uint32_t Offset;
1001  if (!GcovBuffer.readInt(Offset))
1003  InlineCallStack NewStack;
1004  NewStack.push_back(FProfile);
1005  NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end());
1006  if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset))
1007  return EC;
1008  }
1009 
1011 }
1012 
1013 /// Read a GCC AutoFDO profile.
1014 ///
1015 /// This format is generated by the Linux Perf conversion tool at
1016 /// https://github.com/google/autofdo.
1017 std::error_code SampleProfileReaderGCC::read() {
1018  // Read the string table.
1019  if (std::error_code EC = readNameTable())
1020  return EC;
1021 
1022  // Read the source profile.
1023  if (std::error_code EC = readFunctionProfiles())
1024  return EC;
1025 
1027 }
1028 
1030  StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart()));
1031  return Magic == "adcg*704";
1032 }
1033 
1035  // If the underlying data is in compact format, we can't remap it because
1036  // we don't know what the original function names were.
1037  if (getFormat() == SPF_Compact_Binary) {
1039  Buffer->getBufferIdentifier(),
1040  "Profile data remapping cannot be applied to profile data "
1041  "in compact format (original mangled names are not available).",
1042  DS_Warning));
1044  }
1045 
1046  if (Error E = Remappings.read(*Buffer)) {
1048  std::move(E), [&](const SymbolRemappingParseError &ParseError) {
1049  reportError(ParseError.getLineNum(), ParseError.getMessage());
1050  });
1052  }
1053 
1054  for (auto &Sample : getProfiles())
1055  if (auto Key = Remappings.insert(Sample.first()))
1056  SampleMap.insert({Key, &Sample.second});
1057 
1059 }
1060 
1063  if (auto Key = Remappings.lookup(Fname))
1064  return SampleMap.lookup(Key);
1065  return SampleProfileReader::getSamplesFor(Fname);
1066 }
1067 
1068 /// Prepare a memory buffer for the contents of \p Filename.
1069 ///
1070 /// \returns an error code indicating the status of the buffer.
1072 setupMemoryBuffer(const Twine &Filename) {
1073  auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename);
1074  if (std::error_code EC = BufferOrErr.getError())
1075  return EC;
1076  auto Buffer = std::move(BufferOrErr.get());
1077 
1078  // Sanity check the file.
1079  if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint32_t>::max())
1081 
1082  return std::move(Buffer);
1083 }
1084 
1085 /// Create a sample profile reader based on the format of the input file.
1086 ///
1087 /// \param Filename The file to open.
1088 ///
1089 /// \param C The LLVM context to use to emit diagnostics.
1090 ///
1091 /// \returns an error code indicating the status of the created reader.
1094  auto BufferOrError = setupMemoryBuffer(Filename);
1095  if (std::error_code EC = BufferOrError.getError())
1096  return EC;
1097  return create(BufferOrError.get(), C);
1098 }
1099 
1100 /// Create a sample profile remapper from the given input, to remap the
1101 /// function names in the given profile data.
1102 ///
1103 /// \param Filename The file to open.
1104 ///
1105 /// \param C The LLVM context to use to emit diagnostics.
1106 ///
1107 /// \param Underlying The underlying profile data reader to remap.
1108 ///
1109 /// \returns an error code indicating the status of the created reader.
1112  const Twine &Filename, LLVMContext &C,
1113  std::unique_ptr<SampleProfileReader> Underlying) {
1114  auto BufferOrError = setupMemoryBuffer(Filename);
1115  if (std::error_code EC = BufferOrError.getError())
1116  return EC;
1117  return std::make_unique<SampleProfileReaderItaniumRemapper>(
1118  std::move(BufferOrError.get()), C, std::move(Underlying));
1119 }
1120 
1121 /// Create a sample profile reader based on the format of the input data.
1122 ///
1123 /// \param B The memory buffer to create the reader from (assumes ownership).
1124 ///
1125 /// \param C The LLVM context to use to emit diagnostics.
1126 ///
1127 /// \returns an error code indicating the status of the created reader.
1129 SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) {
1130  std::unique_ptr<SampleProfileReader> Reader;
1132  Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));
1134  Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C));
1136  Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C));
1137  else if (SampleProfileReaderGCC::hasFormat(*B))
1138  Reader.reset(new SampleProfileReaderGCC(std::move(B), C));
1140  Reader.reset(new SampleProfileReaderText(std::move(B), C));
1141  else
1143 
1144  FunctionSamples::Format = Reader->getFormat();
1145  if (std::error_code EC = Reader->readHeader()) {
1146  return EC;
1147  }
1148 
1149  return std::move(Reader);
1150 }
1151 
1152 // For text and GCC file formats, we compute the summary after reading the
1153 // profile. Binary format has the profile summary in its header.
1156  for (const auto &I : Profiles) {
1157  const FunctionSamples &Profile = I.second;
1158  Builder.addRecord(Profile);
1159  }
1160  Summary = Builder.getSummary();
1161 }
uint64_t CallInst * C
std::unique_ptr< MemoryBuffer > Buffer
Memory buffer holding the profile file.
Represents either an error or a value T.
Definition: ErrorOr.h:56
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
FunctionSamples * getSamplesFor(StringRef FunctionName) override
Return the samples collected for function F.
std::error_code read() override
Read remapping file and apply it to the sample profile.
void collectFuncsToUse(const Module &M) override
Collect functions to be used when compiling Module M.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:65
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
LLVM_NODISCARD size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
Definition: StringRef.h:345
std::error_code read() override
Read sample profiles from the associated file.
void push_back(const T &Elt)
Definition: SmallVector.h:211
void addRecord(const sampleprof::FunctionSamples &FS, bool isCallsiteSample=false)
std::error_code readSummary()
Read profile summary.
virtual std::error_code readHeader() override
Read and validate the file header.
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string&#39;s MD5 hash.
Definition: MD5.h:109
LLVM_NODISCARD size_t find_last_of(char C, size_t From=npos) const
Find the last character in the string that is C, or npos if not found.
Definition: StringRef.h:406
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(const Twine &Filename, LLVMContext &C)
Create a sample profile reader appropriate to the file format.
std::error_code read() override
Read sample profiles from the associated file.
A forward iterator which reads text lines from a buffer.
Definition: LineIterator.h:31
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:64
std::error_code readProfile(FunctionSamples &FProfile)
Read the contents of the given profile instance.
F(f)
static SampleProfileFormat Format
Definition: SampleProf.h:518
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
Representation of the samples collected for a function.
Definition: SampleProf.h:261
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
virtual std::error_code readNameTable()
Read the whole name table.
StringMap< FunctionSamples > Profiles
Map every function to its associated profile.
Tagged union holding either a T or a Error.
Definition: yaml2obj.h:21
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:578
bool is_at_eof() const
Return true if we&#39;ve reached EOF or are an "end" iterator.
Definition: LineIterator.h:49
Key
PAL metadata keys.
SampleProfileFormat getFormat()
Return the profile format.
static bool ParseHead(const StringRef &Input, StringRef &FName, uint64_t &NumSamples, uint64_t &NumHeadSamples)
Parse Input as function head.
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
void setName(StringRef FunctionName)
Set the name of the function.
Definition: SampleProf.h:448
static uint64_t SPVersion()
Definition: SampleProf.h:108
static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, DenseMap< StringRef, uint64_t > &TargetCountMap)
Parse Input as line sample.
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1231
int64_t line_number() const
Return the current line number. May return any number at EOF.
Definition: LineIterator.h:55
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:131
std::error_code readOneFunctionProfile(const InlineCallStack &InlineStack, bool Update, uint32_t Offset)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
sampleprof_error
Definition: SampleProf.h:41
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
FunctionSamples * getSamplesFor(const Function &F)
Return the samples collected for function F.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator, uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:284
ErrorOr< T > readNumber()
Read a numeric value of type T from the profile.
LLVM_NODISCARD size_t find_first_not_of(char C, size_t From=0) const
Find the first character in the string that is not C or npos if not found.
Definition: StringRef.cpp:249
LLVMContext & Ctx
LLVM context used to emit diagnostics.
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes...
Definition: SampleProf.h:460
LLVM_NODISCARD size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition: StringRef.h:285
std::underlying_type< E >::type Underlying(E Val)
Check that Val is in range for E, and return Val cast to E&#39;s underlying type.
Definition: BitmaskEnum.h:90
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
Definition: LEB128.h:128
ErrorOr< StringRef > readString()
Read a string from the profile.
virtual std::error_code readHeader() override
Read and validate the file header.
std::error_code readFuncProfile()
Read the next function profile instance.
size_t size() const
Definition: SmallVector.h:52
std::error_code readMagicIdent()
Read the contents of Magic number and Version number.
GCOVVersion
Definition: GCOV.h:43
void dumpFunctionProfile(StringRef FName, raw_ostream &OS=dbgs())
Print the profile for FName on stream OS.
static const char *const Magic
Definition: Archive.cpp:41
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:478
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
Definition: Error.h:904
sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:268
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
FunctionSamplesMap & functionSamplesAt(const LineLocation &Loc)
Return the function samples at the given callsite location.
Definition: SampleProf.h:322
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:41
static StringRef readString(WasmObjectFile::ReadContext &Ctx)
Target - Wrapper for Target specific information.
StringMap< FunctionSamples > & getProfiles()
Return all the profiles.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:467
std::unique_ptr< ProfileSummary > getSummary()
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, int64_t FileSize=-1, bool RequiresNullTerminator=true)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
ErrorOr< T > readUnencodedNumber()
Read a numeric value of type T from the profile.
static const size_t npos
Definition: StringRef.h:50
Represents the relative location of an instruction.
Definition: SampleProf.h:140
LLVM_NODISCARD size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:380
#define I(x, y, z)
Definition: MD5.cpp:58
uint32_t Size
Definition: Profile.cpp:46
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
const char * getBufferStart() const
Definition: MemoryBuffer.h:59
static uint64_t SPMagic(SampleProfileFormat Format=SPF_Binary)
Definition: SampleProf.h:91
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:276
const std::string to_string(const T &Value)
Definition: ScopedPrinter.h:61
Provides ErrorOr<T> smart pointer.
std::error_code readSectionTag(uint32_t Expected)
Read the section tag and check that it&#39;s the same as Expected.
static ErrorOr< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename)
Prepare a memory buffer for the contents of Filename.
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
Definition: SampleProf.h:365
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
ErrorOr< uint32_t > readStringIndex(T &Table)
Read the string index and check whether it overflows the table.
void reportError(int64_t LineNumber, Twine Msg) const
Report a parse error message.
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:45
virtual ErrorOr< StringRef > readStringFromTable()
Read a string indirectly via the name table.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
Diagnostic information for the sample profiler.
std::error_code readHeader() override
Read and validate the file header.
void computeSummary()
Compute summary for this profile.
void dump(raw_ostream &OS=dbgs())
Print all the profiles on stream OS.
std::error_code read() override
Read sample profiles from the associated file.
std::error_code read() override
Read sample profiles in extensible format from the associated file.
const uint64_t Version
Definition: InstrProf.h:984
static bool isOffsetLegal(unsigned L)
Returns true if line offset L is legal (only has 16 bits).
sampleprof_error addCalledTargetSamples(uint32_t LineOffset, uint32_t Discriminator, StringRef FName, uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:290
sampleprof_error MergeResult(sampleprof_error &Accumulator, sampleprof_error Result)
Definition: SampleProf.h:60
std::unique_ptr< ProfileSummary > Summary
Profile summary information.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(const Twine &Filename, LLVMContext &C, std::unique_ptr< SampleProfileReader > Underlying)
Create a remapped sample profile from the given remapping file and underlying samples.
std::error_code read() override
Read samples only for functions to use.