Line data Source code
1 : //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file contains support for writing profiling data for clang's
11 : // instrumentation based PGO and coverage.
12 : //
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "llvm/ProfileData/InstrProfWriter.h"
16 : #include "llvm/ADT/STLExtras.h"
17 : #include "llvm/ADT/StringRef.h"
18 : #include "llvm/IR/ProfileSummary.h"
19 : #include "llvm/ProfileData/InstrProf.h"
20 : #include "llvm/ProfileData/ProfileCommon.h"
21 : #include "llvm/Support/Endian.h"
22 : #include "llvm/Support/EndianStream.h"
23 : #include "llvm/Support/Error.h"
24 : #include "llvm/Support/MemoryBuffer.h"
25 : #include "llvm/Support/OnDiskHashTable.h"
26 : #include "llvm/Support/raw_ostream.h"
27 : #include <algorithm>
28 : #include <cstdint>
29 : #include <memory>
30 : #include <string>
31 : #include <tuple>
32 : #include <utility>
33 : #include <vector>
34 :
35 : using namespace llvm;
36 :
37 : // A struct to define how the data stream should be patched. For Indexed
38 : // profiling, only uint64_t data type is needed.
39 : struct PatchItem {
40 : uint64_t Pos; // Where to patch.
41 : uint64_t *D; // Pointer to an array of source data.
42 : int N; // Number of elements in \c D array.
43 : };
44 :
45 : namespace llvm {
46 :
47 : // A wrapper class to abstract writer stream with support of bytes
48 : // back patching.
49 : class ProfOStream {
50 : public:
51 : ProfOStream(raw_fd_ostream &FD)
52 97 : : IsFDOStream(true), OS(FD), LE(FD, support::little) {}
53 : ProfOStream(raw_string_ostream &STR)
54 131 : : IsFDOStream(false), OS(STR), LE(STR, support::little) {}
55 :
56 0 : uint64_t tell() { return OS.tell(); }
57 12996 : void write(uint64_t V) { LE.write<uint64_t>(V); }
58 :
59 : // \c patch can only be called when all data is written and flushed.
60 : // For raw_string_ostream, the patch is done on the target string
61 : // directly and it won't be reflected in the stream's internal buffer.
62 228 : void patch(PatchItem *P, int NItems) {
63 : using namespace support;
64 :
65 228 : if (IsFDOStream) {
66 97 : raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
67 291 : for (int K = 0; K < NItems; K++) {
68 194 : FDOStream.seek(P[K].Pos);
69 5723 : for (int I = 0; I < P[K].N; I++)
70 5529 : write(P[K].D[I]);
71 : }
72 : } else {
73 131 : raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
74 : std::string &Data = SOStream.str(); // with flush
75 393 : for (int K = 0; K < NItems; K++) {
76 7729 : for (int I = 0; I < P[K].N; I++) {
77 7467 : uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
78 7467 : Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
79 7467 : (const char *)&Bytes, sizeof(uint64_t));
80 : }
81 : }
82 : }
83 228 : }
84 :
85 : // If \c OS is an instance of \c raw_fd_ostream, this field will be
86 : // true. Otherwise, \c OS will be an raw_string_ostream.
87 : bool IsFDOStream;
88 : raw_ostream &OS;
89 : support::endian::Writer LE;
90 : };
91 :
92 : class InstrProfRecordWriterTrait {
93 : public:
94 : using key_type = StringRef;
95 : using key_type_ref = StringRef;
96 :
97 : using data_type = const InstrProfWriter::ProfilingData *const;
98 : using data_type_ref = const InstrProfWriter::ProfilingData *const;
99 :
100 : using hash_value_type = uint64_t;
101 : using offset_type = uint64_t;
102 :
103 : support::endianness ValueProfDataEndianness = support::little;
104 : InstrProfSummaryBuilder *SummaryBuilder;
105 :
106 403 : InstrProfRecordWriterTrait() = default;
107 :
108 : static hash_value_type ComputeHash(key_type_ref K) {
109 : return IndexedInstrProf::ComputeHash(K);
110 : }
111 :
112 : static std::pair<offset_type, offset_type>
113 403 : EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
114 : using namespace support;
115 :
116 : endian::Writer LE(Out, little);
117 :
118 : offset_type N = K.size();
119 : LE.write<offset_type>(N);
120 :
121 : offset_type M = 0;
122 1223 : for (const auto &ProfileData : *V) {
123 : const InstrProfRecord &ProfRecord = ProfileData.second;
124 : M += sizeof(uint64_t); // The function hash
125 417 : M += sizeof(uint64_t); // The size of the Counts vector
126 417 : M += ProfRecord.Counts.size() * sizeof(uint64_t);
127 :
128 : // Value data
129 417 : M += ValueProfData::getSize(ProfileData.second);
130 : }
131 : LE.write<offset_type>(M);
132 :
133 403 : return std::make_pair(N, M);
134 : }
135 :
136 0 : void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {
137 403 : Out.write(K.data(), N);
138 0 : }
139 :
140 403 : void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
141 : using namespace support;
142 :
143 : endian::Writer LE(Out, little);
144 1223 : for (const auto &ProfileData : *V) {
145 417 : const InstrProfRecord &ProfRecord = ProfileData.second;
146 417 : SummaryBuilder->addRecord(ProfRecord);
147 :
148 417 : LE.write<uint64_t>(ProfileData.first); // Function hash
149 417 : LE.write<uint64_t>(ProfRecord.Counts.size());
150 1576 : for (uint64_t I : ProfRecord.Counts)
151 : LE.write<uint64_t>(I);
152 :
153 : // Write value data
154 : std::unique_ptr<ValueProfData> VDataPtr =
155 417 : ValueProfData::serializeFrom(ProfileData.second);
156 417 : uint32_t S = VDataPtr->getSize();
157 417 : VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
158 834 : Out.write((const char *)VDataPtr.get(), S);
159 : }
160 403 : }
161 : };
162 :
163 : } // end namespace llvm
164 :
165 278 : InstrProfWriter::InstrProfWriter(bool Sparse)
166 556 : : Sparse(Sparse), InfoObj(new InstrProfRecordWriterTrait()) {}
167 :
168 275 : InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
169 :
170 : // Internal interface for testing purpose only.
171 4 : void InstrProfWriter::setValueProfDataEndianness(
172 : support::endianness Endianness) {
173 4 : InfoObj->ValueProfDataEndianness = Endianness;
174 4 : }
175 :
176 155 : void InstrProfWriter::setOutputSparse(bool Sparse) {
177 155 : this->Sparse = Sparse;
178 155 : }
179 :
180 539 : void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,
181 : function_ref<void(Error)> Warn) {
182 539 : auto Name = I.Name;
183 539 : auto Hash = I.Hash;
184 539 : addRecord(Name, Hash, std::move(I), Weight, Warn);
185 539 : }
186 :
187 583 : void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
188 : InstrProfRecord &&I, uint64_t Weight,
189 : function_ref<void(Error)> Warn) {
190 583 : auto &ProfileDataMap = FunctionData[Name];
191 :
192 : bool NewFunc;
193 : ProfilingData::iterator Where;
194 : std::tie(Where, NewFunc) =
195 583 : ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
196 583 : InstrProfRecord &Dest = Where->second;
197 :
198 : auto MapWarn = [&](instrprof_error E) {
199 : Warn(make_error<InstrProfError>(E));
200 583 : };
201 :
202 583 : if (NewFunc) {
203 : // We've never seen a function with this name and hash, add it.
204 492 : Dest = std::move(I);
205 492 : if (Weight > 1)
206 10 : Dest.scale(Weight, MapWarn);
207 : } else {
208 : // We're updating a function we've seen before.
209 91 : Dest.merge(I, Weight, MapWarn);
210 : }
211 :
212 583 : Dest.sortValueData();
213 583 : }
214 :
215 14 : void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
216 : function_ref<void(Error)> Warn) {
217 63 : for (auto &I : IPW.FunctionData)
218 114 : for (auto &Func : I.getValue())
219 88 : addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
220 14 : }
221 :
222 449 : bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
223 449 : if (!Sparse)
224 : return true;
225 188 : for (const auto &Func : PD) {
226 : const InstrProfRecord &IPR = Func.second;
227 88 : if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
228 76 : return true;
229 : }
230 12 : return false;
231 : }
232 :
233 228 : static void setSummary(IndexedInstrProf::Summary *TheSummary,
234 : ProfileSummary &PS) {
235 : using namespace IndexedInstrProf;
236 :
237 : std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
238 228 : TheSummary->NumSummaryFields = Summary::NumKinds;
239 228 : TheSummary->NumCutoffEntries = Res.size();
240 228 : TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
241 228 : TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
242 228 : TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
243 228 : TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
244 228 : TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
245 228 : TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
246 3876 : for (unsigned I = 0; I < Res.size(); I++)
247 : TheSummary->setEntry(I, Res[I]);
248 228 : }
249 :
250 228 : void InstrProfWriter::writeImpl(ProfOStream &OS) {
251 : using namespace IndexedInstrProf;
252 :
253 456 : OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;
254 :
255 228 : InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
256 228 : InfoObj->SummaryBuilder = &ISB;
257 :
258 : // Populate the hash table generator.
259 871 : for (const auto &I : FunctionData)
260 415 : if (shouldEncodeData(I.getValue()))
261 403 : Generator.insert(I.getKey(), &I.getValue());
262 : // Write the header.
263 : IndexedInstrProf::Header Header;
264 228 : Header.Magic = IndexedInstrProf::Magic;
265 228 : Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
266 228 : if (ProfileKind == PF_IRLevel)
267 30 : Header.Version |= VARIANT_MASK_IR_PROF;
268 228 : Header.Unused = 0;
269 228 : Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
270 228 : Header.HashOffset = 0;
271 : int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
272 :
273 : // Only write out all the fields except 'HashOffset'. We need
274 : // to remember the offset of that field to allow back patching
275 : // later.
276 1140 : for (int I = 0; I < N - 1; I++)
277 912 : OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
278 :
279 : // Save the location of Header.HashOffset field in \c OS.
280 228 : uint64_t HashTableStartFieldOffset = OS.tell();
281 : // Reserve the space for HashOffset field.
282 : OS.write(0);
283 :
284 : // Reserve space to write profile summary data.
285 228 : uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
286 : uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
287 : // Remember the summary offset.
288 228 : uint64_t SummaryOffset = OS.tell();
289 12996 : for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
290 : OS.write(0);
291 :
292 : // Write the hash table.
293 228 : uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
294 :
295 : // Allocate space for data to be serialized out.
296 : std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
297 228 : IndexedInstrProf::allocSummary(SummarySize);
298 : // Compute the Summary and copy the data to the data
299 : // structure to be serialized out (to disk or buffer).
300 228 : std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
301 228 : setSummary(TheSummary.get(), *PS);
302 228 : InfoObj->SummaryBuilder = nullptr;
303 :
304 : // Now do the final patch:
305 228 : PatchItem PatchItems[] = {
306 : // Patch the Header.HashOffset field.
307 : {HashTableStartFieldOffset, &HashTableStart, 1},
308 : // Patch the summary data.
309 : {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
310 228 : (int)(SummarySize / sizeof(uint64_t))}};
311 228 : OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems));
312 228 : }
313 :
314 97 : void InstrProfWriter::write(raw_fd_ostream &OS) {
315 : // Write the hash table.
316 : ProfOStream POS(OS);
317 97 : writeImpl(POS);
318 97 : }
319 :
320 131 : std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
321 : std::string Data;
322 131 : raw_string_ostream OS(Data);
323 : ProfOStream POS(OS);
324 : // Write the hash table.
325 131 : writeImpl(POS);
326 : // Return this in an aligned memory buffer.
327 131 : return MemoryBuffer::getMemBufferCopy(Data);
328 : }
329 :
330 : static const char *ValueProfKindStr[] = {
331 : #define VALUE_PROF_KIND(Enumerator, Value) #Enumerator,
332 : #include "llvm/ProfileData/InstrProfData.inc"
333 : };
334 :
335 28 : void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
336 : const InstrProfRecord &Func,
337 : InstrProfSymtab &Symtab,
338 : raw_fd_ostream &OS) {
339 28 : OS << Name << "\n";
340 28 : OS << "# Func Hash:\n" << Hash << "\n";
341 28 : OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
342 28 : OS << "# Counter Values:\n";
343 93 : for (uint64_t Count : Func.Counts)
344 65 : OS << Count << "\n";
345 :
346 : uint32_t NumValueKinds = Func.getNumValueKinds();
347 28 : if (!NumValueKinds) {
348 24 : OS << "\n";
349 24 : return;
350 : }
351 :
352 8 : OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
353 12 : for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
354 : uint32_t NS = Func.getNumValueSites(VK);
355 8 : if (!NS)
356 : continue;
357 12 : OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
358 12 : OS << "# NumValueSites:\n" << NS << "\n";
359 19 : for (uint32_t S = 0; S < NS; S++) {
360 : uint32_t ND = Func.getNumValueDataForSite(VK, S);
361 13 : OS << ND << "\n";
362 13 : std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
363 43 : for (uint32_t I = 0; I < ND; I++) {
364 30 : if (VK == IPVK_IndirectCallTarget)
365 36 : OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":"
366 12 : << VD[I].Count << "\n";
367 : else
368 36 : OS << VD[I].Value << ":" << VD[I].Count << "\n";
369 : }
370 : }
371 : }
372 :
373 4 : OS << "\n";
374 : }
375 :
376 8 : Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
377 8 : if (ProfileKind == PF_IRLevel)
378 4 : OS << "# IR level Instrumentation Flag\n:ir\n";
379 8 : InstrProfSymtab Symtab;
380 33 : for (const auto &I : FunctionData)
381 17 : if (shouldEncodeData(I.getValue()))
382 34 : if (Error E = Symtab.addFuncName(I.getKey()))
383 : return E;
384 :
385 33 : for (const auto &I : FunctionData)
386 17 : if (shouldEncodeData(I.getValue()))
387 52 : for (const auto &Func : I.getValue())
388 36 : writeRecordInText(I.getKey(), Func.first, Func.second, Symtab, OS);
389 : return Error::success();
390 : }
|
