LLVM  9.0.0svn
InstrProfWriter.cpp
Go to the documentation of this file.
1 //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains support for writing profiling data for clang's
10 // instrumentation based PGO and coverage.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/StringRef.h"
17 #include "llvm/IR/ProfileSummary.h"
20 #include "llvm/Support/Endian.h"
22 #include "llvm/Support/Error.h"
26 #include <algorithm>
27 #include <cstdint>
28 #include <memory>
29 #include <string>
30 #include <tuple>
31 #include <utility>
32 #include <vector>
33 
34 using namespace llvm;
35 
36 // A struct to define how the data stream should be patched. For Indexed
37 // profiling, only uint64_t data type is needed.
38 struct PatchItem {
39  uint64_t Pos; // Where to patch.
40  uint64_t *D; // Pointer to an array of source data.
41  int N; // Number of elements in \c D array.
42 };
43 
44 namespace llvm {
45 
46 // A wrapper class to abstract writer stream with support of bytes
47 // back patching.
48 class ProfOStream {
49 public:
51  : IsFDOStream(true), OS(FD), LE(FD, support::little) {}
53  : IsFDOStream(false), OS(STR), LE(STR, support::little) {}
54 
55  uint64_t tell() { return OS.tell(); }
56  void write(uint64_t V) { LE.write<uint64_t>(V); }
57 
58  // \c patch can only be called when all data is written and flushed.
59  // For raw_string_ostream, the patch is done on the target string
60  // directly and it won't be reflected in the stream's internal buffer.
61  void patch(PatchItem *P, int NItems) {
62  using namespace support;
63 
64  if (IsFDOStream) {
65  raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
66  for (int K = 0; K < NItems; K++) {
67  FDOStream.seek(P[K].Pos);
68  for (int I = 0; I < P[K].N; I++)
69  write(P[K].D[I]);
70  }
71  } else {
72  raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
73  std::string &Data = SOStream.str(); // with flush
74  for (int K = 0; K < NItems; K++) {
75  for (int I = 0; I < P[K].N; I++) {
76  uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
77  Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
78  (const char *)&Bytes, sizeof(uint64_t));
79  }
80  }
81  }
82  }
83 
84  // If \c OS is an instance of \c raw_fd_ostream, this field will be
85  // true. Otherwise, \c OS will be an raw_string_ostream.
89 };
90 
92 public:
95 
98 
99  using hash_value_type = uint64_t;
100  using offset_type = uint64_t;
101 
102  support::endianness ValueProfDataEndianness = support::little;
105 
106  InstrProfRecordWriterTrait() = default;
107 
110  }
111 
112  static std::pair<offset_type, offset_type>
114  using namespace support;
115 
116  endian::Writer LE(Out, little);
117 
118  offset_type N = K.size();
119  LE.write<offset_type>(N);
120 
121  offset_type M = 0;
122  for (const auto &ProfileData : *V) {
123  const InstrProfRecord &ProfRecord = ProfileData.second;
124  M += sizeof(uint64_t); // The function hash
125  M += sizeof(uint64_t); // The size of the Counts vector
126  M += ProfRecord.Counts.size() * sizeof(uint64_t);
127 
128  // Value data
129  M += ValueProfData::getSize(ProfileData.second);
130  }
131  LE.write<offset_type>(M);
132 
133  return std::make_pair(N, M);
134  }
135 
137  Out.write(K.data(), N);
138  }
139 
141  using namespace support;
142 
143  endian::Writer LE(Out, little);
144  for (const auto &ProfileData : *V) {
145  const InstrProfRecord &ProfRecord = ProfileData.second;
146  if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
147  CSSummaryBuilder->addRecord(ProfRecord);
148  else
149  SummaryBuilder->addRecord(ProfRecord);
150 
151  LE.write<uint64_t>(ProfileData.first); // Function hash
152  LE.write<uint64_t>(ProfRecord.Counts.size());
153  for (uint64_t I : ProfRecord.Counts)
154  LE.write<uint64_t>(I);
155 
156  // Write value data
157  std::unique_ptr<ValueProfData> VDataPtr =
158  ValueProfData::serializeFrom(ProfileData.second);
159  uint32_t S = VDataPtr->getSize();
160  VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
161  Out.write((const char *)VDataPtr.get(), S);
162  }
163  }
164 };
165 
166 } // end namespace llvm
167 
169  : Sparse(Sparse), InfoObj(new InstrProfRecordWriterTrait()) {}
170 
172 
173 // Internal interface for testing purpose only.
177 }
178 
180  this->Sparse = Sparse;
181 }
182 
184  function_ref<void(Error)> Warn) {
185  auto Name = I.Name;
186  auto Hash = I.Hash;
187  addRecord(Name, Hash, std::move(I), Weight, Warn);
188 }
189 
190 void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
191  InstrProfRecord &&I, uint64_t Weight,
192  function_ref<void(Error)> Warn) {
193  auto &ProfileDataMap = FunctionData[Name];
194 
195  bool NewFunc;
197  std::tie(Where, NewFunc) =
198  ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
199  InstrProfRecord &Dest = Where->second;
200 
201  auto MapWarn = [&](instrprof_error E) {
202  Warn(make_error<InstrProfError>(E));
203  };
204 
205  if (NewFunc) {
206  // We've never seen a function with this name and hash, add it.
207  Dest = std::move(I);
208  if (Weight > 1)
209  Dest.scale(Weight, MapWarn);
210  } else {
211  // We're updating a function we've seen before.
212  Dest.merge(I, Weight, MapWarn);
213  }
214 
215  Dest.sortValueData();
216 }
217 
219  function_ref<void(Error)> Warn) {
220  for (auto &I : IPW.FunctionData)
221  for (auto &Func : I.getValue())
222  addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
223 }
224 
225 bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
226  if (!Sparse)
227  return true;
228  for (const auto &Func : PD) {
229  const InstrProfRecord &IPR = Func.second;
230  if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
231  return true;
232  }
233  return false;
234 }
235 
236 static void setSummary(IndexedInstrProf::Summary *TheSummary,
237  ProfileSummary &PS) {
238  using namespace IndexedInstrProf;
239 
240  std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
241  TheSummary->NumSummaryFields = Summary::NumKinds;
242  TheSummary->NumCutoffEntries = Res.size();
243  TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
244  TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
245  TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
246  TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
247  TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
248  TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
249  for (unsigned I = 0; I < Res.size(); I++)
250  TheSummary->setEntry(I, Res[I]);
251 }
252 
253 void InstrProfWriter::writeImpl(ProfOStream &OS) {
254  using namespace IndexedInstrProf;
255 
257 
259  InfoObj->SummaryBuilder = &ISB;
261  InfoObj->CSSummaryBuilder = &CSISB;
262 
263  // Populate the hash table generator.
264  for (const auto &I : FunctionData)
265  if (shouldEncodeData(I.getValue()))
266  Generator.insert(I.getKey(), &I.getValue());
267  // Write the header.
271  if (ProfileKind == PF_IRLevel)
272  Header.Version |= VARIANT_MASK_IR_PROF;
273  if (ProfileKind == PF_IRLevelWithCS) {
274  Header.Version |= VARIANT_MASK_IR_PROF;
275  Header.Version |= VARIANT_MASK_CSIR_PROF;
276  }
277  Header.Unused = 0;
278  Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
279  Header.HashOffset = 0;
280  int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
281 
282  // Only write out all the fields except 'HashOffset'. We need
283  // to remember the offset of that field to allow back patching
284  // later.
285  for (int I = 0; I < N - 1; I++)
286  OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
287 
288  // Save the location of Header.HashOffset field in \c OS.
289  uint64_t HashTableStartFieldOffset = OS.tell();
290  // Reserve the space for HashOffset field.
291  OS.write(0);
292 
293  // Reserve space to write profile summary data.
295  uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
296  // Remember the summary offset.
297  uint64_t SummaryOffset = OS.tell();
298  for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
299  OS.write(0);
300  uint64_t CSSummaryOffset = 0;
301  uint64_t CSSummarySize = 0;
302  if (ProfileKind == PF_IRLevelWithCS) {
303  CSSummaryOffset = OS.tell();
304  CSSummarySize = SummarySize / sizeof(uint64_t);
305  for (unsigned I = 0; I < CSSummarySize; I++)
306  OS.write(0);
307  }
308 
309  // Write the hash table.
310  uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
311 
312  // Allocate space for data to be serialized out.
313  std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
314  IndexedInstrProf::allocSummary(SummarySize);
315  // Compute the Summary and copy the data to the data
316  // structure to be serialized out (to disk or buffer).
317  std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
318  setSummary(TheSummary.get(), *PS);
319  InfoObj->SummaryBuilder = nullptr;
320 
321  // For Context Sensitive summary.
322  std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
323  if (ProfileKind == PF_IRLevelWithCS) {
324  TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
325  std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
326  setSummary(TheCSSummary.get(), *CSPS);
327  }
328  InfoObj->CSSummaryBuilder = nullptr;
329 
330  // Now do the final patch:
331  PatchItem PatchItems[] = {
332  // Patch the Header.HashOffset field.
333  {HashTableStartFieldOffset, &HashTableStart, 1},
334  // Patch the summary data.
335  {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
336  (int)(SummarySize / sizeof(uint64_t))},
337  {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
338  (int)CSSummarySize}};
339 
340  OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems));
341 }
342 
344  // Write the hash table.
345  ProfOStream POS(OS);
346  writeImpl(POS);
347 }
348 
349 std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
350  std::string Data;
351  raw_string_ostream OS(Data);
352  ProfOStream POS(OS);
353  // Write the hash table.
354  writeImpl(POS);
355  // Return this in an aligned memory buffer.
356  return MemoryBuffer::getMemBufferCopy(Data);
357 }
358 
359 static const char *ValueProfKindStr[] = {
360 #define VALUE_PROF_KIND(Enumerator, Value) #Enumerator,
362 };
363 
365  const InstrProfRecord &Func,
366  InstrProfSymtab &Symtab,
367  raw_fd_ostream &OS) {
368  OS << Name << "\n";
369  OS << "# Func Hash:\n" << Hash << "\n";
370  OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
371  OS << "# Counter Values:\n";
372  for (uint64_t Count : Func.Counts)
373  OS << Count << "\n";
374 
375  uint32_t NumValueKinds = Func.getNumValueKinds();
376  if (!NumValueKinds) {
377  OS << "\n";
378  return;
379  }
380 
381  OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
382  for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
383  uint32_t NS = Func.getNumValueSites(VK);
384  if (!NS)
385  continue;
386  OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
387  OS << "# NumValueSites:\n" << NS << "\n";
388  for (uint32_t S = 0; S < NS; S++) {
389  uint32_t ND = Func.getNumValueDataForSite(VK, S);
390  OS << ND << "\n";
391  std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
392  for (uint32_t I = 0; I < ND; I++) {
393  if (VK == IPVK_IndirectCallTarget)
394  OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":"
395  << VD[I].Count << "\n";
396  else
397  OS << VD[I].Value << ":" << VD[I].Count << "\n";
398  }
399  }
400  }
401 
402  OS << "\n";
403 }
404 
406  if (ProfileKind == PF_IRLevel)
407  OS << "# IR level Instrumentation Flag\n:ir\n";
408  else if (ProfileKind == PF_IRLevelWithCS)
409  OS << "# CSIR level Instrumentation Flag\n:csir\n";
410  InstrProfSymtab Symtab;
411 
413  using RecordType = std::pair<StringRef, FuncPair>;
414  SmallVector<RecordType, 4> OrderedFuncData;
415 
416  for (const auto &I : FunctionData) {
417  if (shouldEncodeData(I.getValue())) {
418  if (Error E = Symtab.addFuncName(I.getKey()))
419  return E;
420  for (const auto &Func : I.getValue())
421  OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
422  }
423  }
424 
425  llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
426  return std::tie(A.first, A.second.first) <
427  std::tie(B.first, B.second.first);
428  });
429 
430  for (const auto &record : OrderedFuncData) {
431  const StringRef &Name = record.first;
432  const FuncPair &Func = record.second;
433  writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
434  }
435 
436  return Error::success();
437 }
void setValueProfDataEndianness(support::endianness Endianness)
const InstrProfWriter::ProfilingData *const data_type
static const char * ValueProfKindStr[]
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:410
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:535
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:691
This class represents lattice values for constants.
Definition: AllocatorList.h:23
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
ProfOStream(raw_string_ostream &STR)
Defines facilities for reading and writing on-disk hash tables.
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:804
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:993
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
InstrProfWriter(bool Sparse=false)
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:116
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1001
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:64
constexpr support::endianness Endianness
The endianness of all multi-byte encoded values in MessagePack.
Definition: MsgPack.h:24
block Block Frequency true
uint64_t getMaxInternalCount()
void addRecord(const InstrProfRecord &)
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:800
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
const uint64_t Magic
Definition: InstrProf.h:880
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
const InstrProfWriter::ProfilingData *const data_type_ref
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
const HashT HashType
Definition: InstrProf.h:906
void scale(uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by Weight.
Definition: InstrProf.cpp:563
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
uint64_t getMaxFunctionCount()
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
SummaryEntryVector & getDetailedSummary()
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
#define P(N)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
void setOutputSparse(bool Sparse)
static hash_value_type ComputeHash(key_type_ref K)
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
uint64_t Pos
InstrProfSummaryBuilder * CSSummaryBuilder
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
Definition: InstrProf.h:786
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1192
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:778
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:908
static void write(bool isBE, void *P, T V)
std::string & str()
Flushes the stream contents to the target string and returns the string&#39;s reference.
Definition: raw_ostream.h:498
raw_ostream & write(unsigned char C)
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1115
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
void write(uint64_t V)
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
uint64_t getTotalCount()
std::unique_ptr< ProfileSummary > getSummary()
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
support::endianness ValueProfDataEndianness
support::endian::Writer LE
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it...
Generates an on disk hash table.
std::vector< uint64_t > Counts
Definition: InstrProf.h:622
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
Adapter to write values to a stream in a particular byte order.
Definition: EndianStream.h:51
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:365
uint32_t getNumFunctions()
Profiling information for a single function.
Definition: InstrProf.h:621
instrprof_error
Definition: InstrProf.h:279
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:471
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
void patch(PatchItem *P, int NItems)
InstrProfSummaryBuilder * SummaryBuilder
uint64_t * D
LLVM_NODISCARD const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:122
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:482
void write(raw_fd_ostream &OS)
Write the profile to OS.
LLVM Value Representation.
Definition: Value.h:72
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:45
ProfOStream(raw_fd_ostream &FD)
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
Definition: InstrProf.h:810
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
Definition: InstrProf.h:554
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:987
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.