Line data Source code
1 : //===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements the class that reads LLVM sample profiles. It
11 : // supports three file formats: text, binary and gcov.
12 : //
13 : // The textual representation is useful for debugging and testing purposes. The
14 : // binary representation is more compact, resulting in smaller file sizes.
15 : //
16 : // The gcov encoding is the one generated by GCC's AutoFDO profile creation
17 : // tool (https://github.com/google/autofdo)
18 : //
19 : // All three encodings can be used interchangeably as an input sample profile.
20 : //
21 : //===----------------------------------------------------------------------===//
22 :
23 : #include "llvm/ProfileData/SampleProfReader.h"
24 : #include "llvm/ADT/DenseMap.h"
25 : #include "llvm/ADT/STLExtras.h"
26 : #include "llvm/ADT/StringRef.h"
27 : #include "llvm/IR/ProfileSummary.h"
28 : #include "llvm/ProfileData/ProfileCommon.h"
29 : #include "llvm/ProfileData/SampleProf.h"
30 : #include "llvm/Support/ErrorOr.h"
31 : #include "llvm/Support/LEB128.h"
32 : #include "llvm/Support/LineIterator.h"
33 : #include "llvm/Support/MD5.h"
34 : #include "llvm/Support/MemoryBuffer.h"
35 : #include "llvm/Support/raw_ostream.h"
36 : #include <algorithm>
37 : #include <cstddef>
38 : #include <cstdint>
39 : #include <limits>
40 : #include <memory>
41 : #include <system_error>
42 : #include <vector>
43 :
44 : using namespace llvm;
45 : using namespace sampleprof;
46 :
47 : /// Dump the function profile for \p FName.
48 : ///
49 : /// \param FName Name of the function to print.
50 : /// \param OS Stream to emit the output to.
51 22 : void SampleProfileReader::dumpFunctionProfile(StringRef FName,
52 : raw_ostream &OS) {
53 22 : OS << "Function: " << FName << ": " << Profiles[FName];
54 22 : }
55 :
56 : /// Dump all the function profiles found on stream \p OS.
57 11 : void SampleProfileReader::dump(raw_ostream &OS) {
58 43 : for (const auto &I : Profiles)
59 21 : dumpFunctionProfile(I.getKey(), OS);
60 11 : }
61 :
62 : /// Parse \p Input as function head.
63 : ///
64 : /// Parse one line of \p Input, and update function name in \p FName,
65 : /// function's total sample count in \p NumSamples, function's entry
66 : /// count in \p NumHeadSamples.
67 : ///
68 : /// \returns true if parsing is successful.
69 251 : static bool ParseHead(const StringRef &Input, StringRef &FName,
70 : uint64_t &NumSamples, uint64_t &NumHeadSamples) {
71 502 : if (Input[0] == ' ')
72 : return false;
73 251 : size_t n2 = Input.rfind(':');
74 502 : size_t n1 = Input.rfind(':', n2 - 1);
75 251 : FName = Input.substr(0, n1);
76 502 : if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples))
77 3 : return false;
78 742 : if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples))
79 2 : return false;
80 : return true;
81 : }
82 :
83 : /// Returns true if line offset \p L is legal (only has 16 bits).
84 604 : static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
85 :
86 : /// Parse \p Input as line sample.
87 : ///
88 : /// \param Input input line.
89 : /// \param IsCallsite true if the line represents an inlined callsite.
90 : /// \param Depth the depth of the inline stack.
91 : /// \param NumSamples total samples of the line/inlined callsite.
92 : /// \param LineOffset line offset to the start of the function.
93 : /// \param Discriminator discriminator of the line.
94 : /// \param TargetCountMap map from indirect call target to count.
95 : ///
96 : /// returns true if parsing is successful.
97 602 : static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth,
98 : uint64_t &NumSamples, uint32_t &LineOffset,
99 : uint32_t &Discriminator, StringRef &CalleeName,
100 : DenseMap<StringRef, uint64_t> &TargetCountMap) {
101 1407 : for (Depth = 0; Input[Depth] == ' '; Depth++)
102 : ;
103 602 : if (Depth == 0)
104 : return false;
105 :
106 : size_t n1 = Input.find(':');
107 1204 : StringRef Loc = Input.substr(Depth, n1 - Depth);
108 443 : size_t n2 = Loc.find('.');
109 159 : if (n2 == StringRef::npos) {
110 443 : if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset))
111 0 : return false;
112 443 : Discriminator = 0;
113 : } else {
114 159 : if (Loc.substr(0, n2).getAsInteger(10, LineOffset))
115 0 : return false;
116 477 : if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator))
117 2 : return false;
118 : }
119 :
120 1200 : StringRef Rest = Input.substr(n1 + 2);
121 600 : if (Rest[0] >= '0' && Rest[0] <= '9') {
122 514 : IsCallsite = false;
123 471 : size_t n3 = Rest.find(' ');
124 43 : if (n3 == StringRef::npos) {
125 471 : if (Rest.getAsInteger(10, NumSamples))
126 0 : return false;
127 : } else {
128 86 : if (Rest.substr(0, n3).getAsInteger(10, NumSamples))
129 0 : return false;
130 : }
131 : // Find call targets and their sample counts.
132 : // Note: In some cases, there are symbols in the profile which are not
133 : // mangled. To accommodate such cases, use colon + integer pairs as the
134 : // anchor points.
135 : // An example:
136 : // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
137 : // ":1000" and ":437" are used as anchor points so the string above will
138 : // be interpreted as
139 : // target: _M_construct<char *>
140 : // count: 1000
141 : // target: string_view<std::allocator<char> >
142 : // count: 437
143 526 : while (n3 != StringRef::npos) {
144 55 : n3 += Rest.substr(n3).find_first_not_of(' ');
145 55 : Rest = Rest.substr(n3);
146 55 : n3 = Rest.find_first_of(':');
147 55 : if (n3 == StringRef::npos || n3 == 0)
148 0 : return false;
149 :
150 : StringRef Target;
151 : uint64_t count, n4;
152 : while (true) {
153 : // Get the segment after the current colon.
154 118 : StringRef AfterColon = Rest.substr(n3 + 1);
155 : // Get the target symbol before the current colon.
156 59 : Target = Rest.substr(0, n3);
157 : // Check if the word after the current colon is an integer.
158 59 : n4 = AfterColon.find_first_of(' ');
159 59 : n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
160 118 : StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1);
161 59 : if (!WordAfterColon.getAsInteger(10, count))
162 : break;
163 :
164 : // Try to find the next colon.
165 4 : uint64_t n5 = AfterColon.find_first_of(':');
166 4 : if (n5 == StringRef::npos)
167 0 : return false;
168 4 : n3 += n5 + 1;
169 4 : }
170 :
171 : // An anchor point is found. Save the {target, count} pair
172 55 : TargetCountMap[Target] = count;
173 55 : if (n4 == Rest.size())
174 : break;
175 : // Change n3 to the next blank space after colon + integer pair.
176 : n3 = n4;
177 : }
178 : } else {
179 86 : IsCallsite = true;
180 : size_t n3 = Rest.find_last_of(':');
181 86 : CalleeName = Rest.substr(0, n3);
182 254 : if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples))
183 4 : return false;
184 : }
185 : return true;
186 : }
187 :
188 : /// Load samples from a text file.
189 : ///
190 : /// See the documentation at the top of the file for an explanation of
191 : /// the expected format.
192 : ///
193 : /// \returns true if the file was loaded successfully, false otherwise.
194 99 : std::error_code SampleProfileReaderText::read() {
195 99 : line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
196 : sampleprof_error Result = sampleprof_error::success;
197 :
198 : InlineCallStack InlineStack;
199 :
200 846 : for (; !LineIt.is_at_eof(); ++LineIt) {
201 755 : if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#')
202 : continue;
203 : // Read the header of each function.
204 : //
205 : // Note that for function identifiers we are actually expecting
206 : // mangled names, but we may not always get them. This happens when
207 : // the compiler decides not to emit the function (e.g., it was inlined
208 : // and removed). In this case, the binary will not have the linkage
209 : // name for the function, so the profiler will emit the function's
210 : // unmangled name, which may contain characters like ':' and '>' in its
211 : // name (member functions, templates, etc).
212 : //
213 : // The only requirement we place on the identifier, then, is that it
214 : // should not begin with a number.
215 751 : if ((*LineIt)[0] != ' ') {
216 : uint64_t NumSamples, NumHeadSamples;
217 149 : StringRef FName;
218 149 : if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) {
219 4 : reportError(LineIt.line_number(),
220 2 : "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
221 0 : return sampleprof_error::malformed;
222 : }
223 294 : Profiles[FName] = FunctionSamples();
224 147 : FunctionSamples &FProfile = Profiles[FName];
225 : FProfile.setName(FName);
226 147 : MergeResult(Result, FProfile.addTotalSamples(NumSamples));
227 147 : MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
228 : InlineStack.clear();
229 147 : InlineStack.push_back(&FProfile);
230 : } else {
231 : uint64_t NumSamples;
232 602 : StringRef FName;
233 : DenseMap<StringRef, uint64_t> TargetCountMap;
234 : bool IsCallsite;
235 : uint32_t Depth, LineOffset, Discriminator;
236 602 : if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset,
237 : Discriminator, FName, TargetCountMap)) {
238 12 : reportError(LineIt.line_number(),
239 : "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
240 6 : *LineIt);
241 0 : return sampleprof_error::malformed;
242 : }
243 596 : if (IsCallsite) {
244 114 : while (InlineStack.size() > Depth) {
245 : InlineStack.pop_back();
246 : }
247 : FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
248 82 : LineLocation(LineOffset, Discriminator))[FName];
249 : FSamples.setName(FName);
250 82 : MergeResult(Result, FSamples.addTotalSamples(NumSamples));
251 82 : InlineStack.push_back(&FSamples);
252 : } else {
253 1028 : while (InlineStack.size() > Depth) {
254 : InlineStack.pop_back();
255 : }
256 514 : FunctionSamples &FProfile = *InlineStack.back();
257 1083 : for (const auto &name_count : TargetCountMap) {
258 55 : MergeResult(Result, FProfile.addCalledTargetSamples(
259 : LineOffset, Discriminator, name_count.first,
260 55 : name_count.second));
261 : }
262 514 : MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
263 : NumSamples));
264 : }
265 : }
266 : }
267 91 : if (Result == sampleprof_error::success)
268 91 : computeSummary();
269 :
270 91 : return Result;
271 : }
272 :
273 102 : bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
274 : bool result = false;
275 :
276 : // Check that the first non-comment line is a valid function header.
277 102 : line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
278 102 : if (!LineIt.is_at_eof()) {
279 102 : if ((*LineIt)[0] != ' ') {
280 : uint64_t NumSamples, NumHeadSamples;
281 102 : StringRef FName;
282 102 : result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples);
283 : }
284 : }
285 :
286 102 : return result;
287 : }
288 :
289 2256 : template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
290 2256 : unsigned NumBytesRead = 0;
291 : std::error_code EC;
292 2256 : uint64_t Val = decodeULEB128(Data, &NumBytesRead);
293 :
294 462 : if (Val > std::numeric_limits<T>::max())
295 : EC = sampleprof_error::malformed;
296 2256 : else if (Data + NumBytesRead > End)
297 : EC = sampleprof_error::truncated;
298 : else
299 : EC = sampleprof_error::success;
300 :
301 2256 : if (EC) {
302 0 : reportError(0, EC.message());
303 0 : return EC;
304 : }
305 :
306 2256 : Data += NumBytesRead;
307 462 : return static_cast<T>(Val);
308 : }
309 462 :
310 462 : ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
311 : std::error_code EC;
312 462 : StringRef Str(reinterpret_cast<const char *>(Data));
313 : if (Data + Str.size() + 1 > End) {
314 462 : EC = sampleprof_error::truncated;
315 : reportError(0, EC.message());
316 462 : return EC;
317 : }
318 :
319 : Data += Str.size() + 1;
320 : return Str;
321 462 : }
322 0 :
323 0 : template <typename T>
324 : ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
325 : std::error_code EC;
326 462 :
327 462 : if (Data + sizeof(T) > End) {
328 : EC = sampleprof_error::truncated;
329 1794 : reportError(0, EC.message());
330 1794 : return EC;
331 : }
332 1794 :
333 : using namespace support;
334 : T Val = endian::readNext<T, little, unaligned>(Data);
335 : return Val;
336 1794 : }
337 :
338 : template <typename T>
339 : inline ErrorOr<uint32_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
340 : std::error_code EC;
341 1794 : auto Idx = readNumber<uint32_t>();
342 0 : if (std::error_code EC = Idx.getError())
343 0 : return EC;
344 : if (*Idx >= Table.size())
345 : return sampleprof_error::truncated_name_table;
346 1794 : return *Idx;
347 : }
348 :
349 : ErrorOr<StringRef> SampleProfileReaderRawBinary::readStringFromTable() {
350 45 : auto Idx = readStringIndex(NameTable);
351 : if (std::error_code EC = Idx.getError())
352 45 : return EC;
353 45 :
354 : return NameTable[*Idx];
355 0 : }
356 0 :
357 : ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() {
358 : auto Idx = readStringIndex(NameTable);
359 45 : if (std::error_code EC = Idx.getError())
360 : return EC;
361 :
362 : return StringRef(NameTable[*Idx]);
363 : }
364 5 :
365 : std::error_code
366 : SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
367 5 : auto NumSamples = readNumber<uint64_t>();
368 : if (std::error_code EC = NumSamples.getError())
369 0 : return EC;
370 0 : FProfile.addTotalSamples(*NumSamples);
371 :
372 : // Read the samples in the body.
373 : auto NumRecords = readNumber<uint32_t>();
374 : if (std::error_code EC = NumRecords.getError())
375 : return EC;
376 :
377 : for (uint32_t I = 0; I < *NumRecords; ++I) {
378 : auto LineOffset = readNumber<uint64_t>();
379 122 : if (std::error_code EC = LineOffset.getError())
380 : return EC;
381 122 :
382 122 : if (!isOffsetLegal(*LineOffset)) {
383 : return std::error_code();
384 244 : }
385 :
386 : auto Discriminator = readNumber<uint64_t>();
387 : if (std::error_code EC = Discriminator.getError())
388 55 : return EC;
389 :
390 55 : auto NumSamples = readNumber<uint64_t>();
391 55 : if (std::error_code EC = NumSamples.getError())
392 : return EC;
393 110 :
394 : auto NumCalls = readNumber<uint32_t>();
395 : if (std::error_code EC = NumCalls.getError())
396 : return EC;
397 67 :
398 : for (uint32_t J = 0; J < *NumCalls; ++J) {
399 67 : auto CalledFunction(readStringFromTable());
400 67 : if (std::error_code EC = CalledFunction.getError())
401 : return EC;
402 134 :
403 : auto CalledFunctionSamples = readNumber<uint64_t>();
404 : if (std::error_code EC = CalledFunctionSamples.getError())
405 : return EC;
406 :
407 67 : FProfile.addCalledTargetSamples(*LineOffset, *Discriminator,
408 67 : *CalledFunction, *CalledFunctionSamples);
409 67 : }
410 :
411 : FProfile.addBodySamples(*LineOffset, *Discriminator, *NumSamples);
412 67 : }
413 :
414 : // Read all the samples for inlined function calls.
415 55 : auto NumCallsites = readNumber<uint32_t>();
416 55 : if (std::error_code EC = NumCallsites.getError())
417 55 : return EC;
418 :
419 : for (uint32_t J = 0; J < *NumCallsites; ++J) {
420 55 : auto LineOffset = readNumber<uint64_t>();
421 : if (std::error_code EC = LineOffset.getError())
422 : return EC;
423 :
424 83 : auto Discriminator = readNumber<uint64_t>();
425 83 : if (std::error_code EC = Discriminator.getError())
426 83 : return EC;
427 0 :
428 83 : auto FName(readStringFromTable());
429 : if (std::error_code EC = FName.getError())
430 : return EC;
431 83 :
432 83 : FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
433 0 : LineLocation(*LineOffset, *Discriminator))[*FName];
434 : CalleeProfile.setName(*FName);
435 244 : if (std::error_code EC = readProfile(CalleeProfile))
436 161 : return EC;
437 161 : }
438 0 :
439 : return sampleprof_error::success;
440 161 : }
441 0 :
442 : std::error_code SampleProfileReaderBinary::readFuncProfile() {
443 : auto NumHeadSamples = readNumber<uint64_t>();
444 161 : if (std::error_code EC = NumHeadSamples.getError())
445 161 : return EC;
446 0 :
447 : auto FName(readStringFromTable());
448 161 : if (std::error_code EC = FName.getError())
449 161 : return EC;
450 0 :
451 : Profiles[*FName] = FunctionSamples();
452 161 : FunctionSamples &FProfile = Profiles[*FName];
453 161 : FProfile.setName(*FName);
454 0 :
455 : FProfile.addHeadSamples(*NumHeadSamples);
456 185 :
457 24 : if (std::error_code EC = readProfile(FProfile))
458 24 : return EC;
459 0 : return sampleprof_error::success;
460 : }
461 24 :
462 24 : std::error_code SampleProfileReaderBinary::read() {
463 0 : while (!at_eof()) {
464 : if (std::error_code EC = readFuncProfile())
465 24 : return EC;
466 : }
467 :
468 : return sampleprof_error::success;
469 161 : }
470 :
471 : std::error_code SampleProfileReaderCompactBinary::read() {
472 : for (auto Name : FuncsToUse) {
473 83 : auto GUID = std::to_string(MD5Hash(Name));
474 83 : auto iter = FuncOffsetTable.find(StringRef(GUID));
475 0 : if (iter == FuncOffsetTable.end())
476 : continue;
477 122 : const uint8_t *SavedData = Data;
478 39 : Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
479 39 : iter->second;
480 0 : if (std::error_code EC = readFuncProfile())
481 : return EC;
482 39 : Data = SavedData;
483 39 : }
484 0 : return sampleprof_error::success;
485 : }
486 39 :
487 39 : std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
488 0 : if (Magic == SPMagic())
489 : return sampleprof_error::success;
490 : return sampleprof_error::bad_magic;
491 39 : }
492 :
493 39 : std::error_code
494 0 : SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) {
495 : if (Magic == SPMagic(SPF_Compact_Binary))
496 : return sampleprof_error::success;
497 83 : return sampleprof_error::bad_magic;
498 : }
499 :
500 44 : std::error_code SampleProfileReaderRawBinary::readNameTable() {
501 44 : auto Size = readNumber<uint32_t>();
502 44 : if (std::error_code EC = Size.getError())
503 0 : return EC;
504 : NameTable.reserve(*Size);
505 44 : for (uint32_t I = 0; I < *Size; ++I) {
506 44 : auto Name(readString());
507 0 : if (std::error_code EC = Name.getError())
508 : return EC;
509 88 : NameTable.push_back(*Name);
510 44 : }
511 :
512 : return sampleprof_error::success;
513 44 : }
514 :
515 44 : std::error_code SampleProfileReaderCompactBinary::readNameTable() {
516 0 : auto Size = readNumber<uint64_t>();
517 44 : if (std::error_code EC = Size.getError())
518 : return EC;
519 : NameTable.reserve(*Size);
520 13 : for (uint32_t I = 0; I < *Size; ++I) {
521 42 : auto FID = readNumber<uint64_t>();
522 29 : if (std::error_code EC = FID.getError())
523 0 : return EC;
524 : NameTable.push_back(std::to_string(*FID));
525 : }
526 13 : return sampleprof_error::success;
527 : }
528 :
529 5 : std::error_code SampleProfileReaderBinary::readHeader() {
530 34 : Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
531 29 : End = Data + Buffer->getBufferSize();
532 58 :
533 29 : // Read and check the magic identifier.
534 : auto Magic = readNumber<uint64_t>();
535 15 : if (std::error_code EC = Magic.getError())
536 30 : return EC;
537 15 : else if (std::error_code EC = verifySPMagic(*Magic))
538 15 : return EC;
539 0 :
540 15 : // Read the version number.
541 : auto Version = readNumber<uint64_t>();
542 5 : if (std::error_code EC = Version.getError())
543 : return EC;
544 : else if (*Version != SPVersion())
545 13 : return sampleprof_error::unsupported_version;
546 13 :
547 13 : if (std::error_code EC = readSummary())
548 0 : return EC;
549 :
550 : if (std::error_code EC = readNameTable())
551 : return EC;
552 5 : return sampleprof_error::success;
553 5 : }
554 5 :
555 0 : std::error_code SampleProfileReaderCompactBinary::readHeader() {
556 : SampleProfileReaderBinary::readHeader();
557 : if (std::error_code EC = readFuncOffsetTable())
558 13 : return EC;
559 13 : return sampleprof_error::success;
560 13 : }
561 0 :
562 13 : std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() {
563 58 : auto TableOffset = readUnencodedNumber<uint64_t>();
564 45 : if (std::error_code EC = TableOffset.getError())
565 45 : return EC;
566 0 :
567 45 : const uint8_t *SavedData = Data;
568 : const uint8_t *TableStart =
569 : reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
570 13 : *TableOffset;
571 : Data = TableStart;
572 :
573 5 : auto Size = readNumber<uint64_t>();
574 5 : if (std::error_code EC = Size.getError())
575 5 : return EC;
576 0 :
577 5 : FuncOffsetTable.reserve(*Size);
578 42 : for (uint32_t I = 0; I < *Size; ++I) {
579 37 : auto FName(readStringFromTable());
580 37 : if (std::error_code EC = FName.getError())
581 0 : return EC;
582 74 :
583 : auto Offset = readNumber<uint64_t>();
584 5 : if (std::error_code EC = Offset.getError())
585 : return EC;
586 :
587 18 : FuncOffsetTable[*FName] = *Offset;
588 18 : }
589 18 : End = TableStart;
590 : Data = SavedData;
591 : return sampleprof_error::success;
592 18 : }
593 18 :
594 0 : void SampleProfileReaderCompactBinary::collectFuncsToUse(const Module &M) {
595 18 : FuncsToUse.clear();
596 0 : for (auto &F : M) {
597 : StringRef Fname = F.getName().split('.').first;
598 : FuncsToUse.insert(Fname);
599 18 : }
600 18 : }
601 0 :
602 18 : std::error_code SampleProfileReaderBinary::readSummaryEntry(
603 0 : std::vector<ProfileSummaryEntry> &Entries) {
604 : auto Cutoff = readNumber<uint64_t>();
605 18 : if (std::error_code EC = Cutoff.getError())
606 0 : return EC;
607 :
608 18 : auto MinBlockCount = readNumber<uint64_t>();
609 0 : if (std::error_code EC = MinBlockCount.getError())
610 18 : return EC;
611 :
612 : auto NumBlocks = readNumber<uint64_t>();
613 5 : if (std::error_code EC = NumBlocks.getError())
614 5 : return EC;
615 5 :
616 0 : Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
617 5 : return sampleprof_error::success;
618 : }
619 :
620 5 : std::error_code SampleProfileReaderBinary::readSummary() {
621 5 : auto TotalCount = readNumber<uint64_t>();
622 5 : if (std::error_code EC = TotalCount.getError())
623 0 : return EC;
624 :
625 5 : auto MaxBlockCount = readNumber<uint64_t>();
626 : if (std::error_code EC = MaxBlockCount.getError())
627 5 : return EC;
628 5 :
629 5 : auto MaxFunctionCount = readNumber<uint64_t>();
630 : if (std::error_code EC = MaxFunctionCount.getError())
631 5 : return EC;
632 5 :
633 0 : auto NumBlocks = readNumber<uint64_t>();
634 : if (std::error_code EC = NumBlocks.getError())
635 5 : return EC;
636 20 :
637 15 : auto NumFunctions = readNumber<uint64_t>();
638 15 : if (std::error_code EC = NumFunctions.getError())
639 0 : return EC;
640 :
641 15 : auto NumSummaryEntries = readNumber<uint64_t>();
642 15 : if (std::error_code EC = NumSummaryEntries.getError())
643 0 : return EC;
644 :
645 15 : std::vector<ProfileSummaryEntry> Entries;
646 : for (unsigned i = 0; i < *NumSummaryEntries; i++) {
647 5 : std::error_code EC = readSummaryEntry(Entries);
648 5 : if (EC != sampleprof_error::success)
649 5 : return EC;
650 : }
651 : Summary = llvm::make_unique<ProfileSummary>(
652 5 : ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0,
653 : *MaxFunctionCount, *NumBlocks, *NumFunctions);
654 36 :
655 62 : return sampleprof_error::success;
656 : }
657 :
658 5 : bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
659 : const uint8_t *Data =
660 292 : reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
661 : uint64_t Magic = decodeULEB128(Data);
662 292 : return Magic == SPMagic();
663 292 : }
664 0 :
665 : bool SampleProfileReaderCompactBinary::hasFormat(const MemoryBuffer &Buffer) {
666 292 : const uint8_t *Data =
667 292 : reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
668 0 : uint64_t Magic = decodeULEB128(Data);
669 : return Magic == SPMagic(SPF_Compact_Binary);
670 292 : }
671 292 :
672 0 : std::error_code SampleProfileReaderGCC::skipNextWord() {
673 : uint32_t dummy;
674 292 : if (!GcovBuffer.readInt(dummy))
675 292 : return sampleprof_error::truncated;
676 : return sampleprof_error::success;
677 : }
678 18 :
679 18 : template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
680 18 : if (sizeof(T) <= sizeof(uint32_t)) {
681 0 : uint32_t Val;
682 : if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
683 18 : return static_cast<T>(Val);
684 18 : } else if (sizeof(T) <= sizeof(uint64_t)) {
685 0 : uint64_t Val;
686 : if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
687 18 : return static_cast<T>(Val);
688 18 : }
689 0 :
690 : std::error_code EC = sampleprof_error::malformed;
691 18 : reportError(0, EC.message());
692 18 : return EC;
693 0 : }
694 :
695 18 : ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
696 18 : StringRef Str;
697 0 : if (!GcovBuffer.readString(Str))
698 : return sampleprof_error::truncated;
699 18 : return Str;
700 18 : }
701 0 :
702 : std::error_code SampleProfileReaderGCC::readHeader() {
703 : // Read the magic identifier.
704 310 : if (!GcovBuffer.readGCDAFormat())
705 292 : return sampleprof_error::unrecognized_format;
706 :
707 0 : // Read the version number. Note - the GCC reader does not validate this
708 : // version, but the profile creator generates v704.
709 36 : GCOV::GCOVVersion version;
710 36 : if (!GcovBuffer.readGCOVVersion(version))
711 : return sampleprof_error::unrecognized_format;
712 :
713 18 : if (version != GCOV::V704)
714 : return sampleprof_error::unsupported_version;
715 :
716 128 : // Skip the empty integer.
717 : if (std::error_code EC = skipNextWord())
718 128 : return EC;
719 :
720 128 : return sampleprof_error::success;
721 : }
722 :
723 115 : std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
724 : uint32_t Tag;
725 115 : if (!GcovBuffer.readInt(Tag))
726 : return sampleprof_error::truncated;
727 115 :
728 : if (Tag != Expected)
729 : return sampleprof_error::malformed;
730 24 :
731 : if (std::error_code EC = skipNextWord())
732 24 : return EC;
733 0 :
734 24 : return sampleprof_error::success;
735 : }
736 :
737 : std::error_code SampleProfileReaderGCC::readNameTable() {
738 : if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames))
739 : return EC;
740 :
741 : uint32_t Size;
742 : if (!GcovBuffer.readInt(Size))
743 : return sampleprof_error::truncated;
744 :
745 : for (uint32_t I = 0; I < Size; ++I) {
746 : StringRef Str;
747 : if (!GcovBuffer.readString(Str))
748 : return sampleprof_error::truncated;
749 : Names.push_back(Str);
750 : }
751 :
752 : return sampleprof_error::success;
753 0 : }
754 0 :
755 0 : std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
756 : if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction))
757 : return EC;
758 :
759 : uint32_t NumFunctions;
760 8 : if (!GcovBuffer.readInt(NumFunctions))
761 : return sampleprof_error::truncated;
762 8 :
763 0 : InlineCallStack Stack;
764 : for (uint32_t I = 0; I < NumFunctions; ++I)
765 : if (std::error_code EC = readOneFunctionProfile(Stack, true, 0))
766 : return EC;
767 :
768 8 : computeSummary();
769 0 : return sampleprof_error::success;
770 : }
771 8 :
772 0 : std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
773 : const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
774 : uint64_t HeadCount = 0;
775 8 : if (InlineStack.size() == 0)
776 0 : if (!GcovBuffer.readInt64(HeadCount))
777 : return sampleprof_error::truncated;
778 8 :
779 : uint32_t NameIdx;
780 : if (!GcovBuffer.readInt(NameIdx))
781 16 : return sampleprof_error::truncated;
782 :
783 16 : StringRef Name(Names[NameIdx]);
784 0 :
785 : uint32_t NumPosCounts;
786 16 : if (!GcovBuffer.readInt(NumPosCounts))
787 0 : return sampleprof_error::truncated;
788 :
789 16 : uint32_t NumCallsites;
790 0 : if (!GcovBuffer.readInt(NumCallsites))
791 : return sampleprof_error::truncated;
792 16 :
793 : FunctionSamples *FProfile = nullptr;
794 : if (InlineStack.size() == 0) {
795 8 : // If this is a top function that we have already processed, do not
796 8 : // update its profile again. This happens in the presence of
797 0 : // function aliases. Since these aliases share the same function
798 : // body, there will be identical replicated profiles for the
799 : // original function. In this case, we simply not bother updating
800 8 : // the profile of the original function.
801 0 : FProfile = &Profiles[Name];
802 : FProfile->addHeadSamples(HeadCount);
803 38 : if (FProfile->getTotalSamples() > 0)
804 30 : Update = false;
805 30 : } else {
806 0 : // Otherwise, we are reading an inlined instance. The top of the
807 60 : // inline stack contains the profile of the caller. Insert this
808 : // callee in the caller's CallsiteMap.
809 : FunctionSamples *CallerProfile = InlineStack.front();
810 8 : uint32_t LineOffset = Offset >> 16;
811 : uint32_t Discriminator = Offset & 0xffff;
812 : FProfile = &CallerProfile->functionSamplesAt(
813 8 : LineLocation(LineOffset, Discriminator))[Name];
814 8 : }
815 0 : FProfile->setName(Name);
816 :
817 : for (uint32_t I = 0; I < NumPosCounts; ++I) {
818 8 : uint32_t Offset;
819 0 : if (!GcovBuffer.readInt(Offset))
820 : return sampleprof_error::truncated;
821 :
822 18 : uint32_t NumTargets;
823 10 : if (!GcovBuffer.readInt(NumTargets))
824 0 : return sampleprof_error::truncated;
825 :
826 8 : uint64_t Count;
827 8 : if (!GcovBuffer.readInt64(Count))
828 : return sampleprof_error::truncated;
829 :
830 30 : // The line location is encoded in the offset as:
831 : // high 16 bits: line offset to the start of the function.
832 30 : // low 16 bits: discriminator.
833 30 : uint32_t LineOffset = Offset >> 16;
834 10 : uint32_t Discriminator = Offset & 0xffff;
835 0 :
836 : InlineCallStack NewStack;
837 : NewStack.push_back(FProfile);
838 30 : NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end());
839 0 : if (Update) {
840 : // Walk up the inline stack, adding the samples on this line to
841 30 : // the total sample count of the callers in the chain.
842 : for (auto CallerProfile : NewStack)
843 : CallerProfile->addTotalSamples(Count);
844 30 :
845 0 : // Update the body samples for the current profile.
846 : FProfile->addBodySamples(LineOffset, Discriminator, Count);
847 : }
848 30 :
849 0 : // Process the list of functions called at an indirect call site.
850 : // These are all the targets that a function pointer (or virtual
851 30 : // function) resolved at runtime.
852 30 : for (uint32_t J = 0; J < NumTargets; J++) {
853 : uint32_t HistVal;
854 : if (!GcovBuffer.readInt(HistVal))
855 : return sampleprof_error::truncated;
856 :
857 : if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
858 : return sampleprof_error::malformed;
859 10 :
860 10 : uint64_t TargetIdx;
861 10 : if (!GcovBuffer.readInt64(TargetIdx))
862 : return sampleprof_error::truncated;
863 : StringRef TargetName(Names[TargetIdx]);
864 :
865 : uint64_t TargetCount;
866 : if (!GcovBuffer.readInt64(TargetCount))
867 20 : return sampleprof_error::truncated;
868 20 :
869 20 : if (Update)
870 20 : FProfile->addCalledTargetSamples(LineOffset, Discriminator,
871 20 : TargetName, TargetCount);
872 : }
873 30 : }
874 :
875 144 : // Process all the inlined callers into the current function. These
876 : // are all the callsites that were inlined into this function.
877 114 : for (uint32_t I = 0; I < NumCallsites; I++) {
878 0 : // The offset is encoded as:
879 : // high 16 bits: line offset to the start of the function.
880 : // low 16 bits: discriminator.
881 114 : uint32_t Offset;
882 0 : if (!GcovBuffer.readInt(Offset))
883 : return sampleprof_error::truncated;
884 : InlineCallStack NewStack;
885 114 : NewStack.push_back(FProfile);
886 0 : NewStack.insert(NewStack.end(), InlineStack.begin(), InlineStack.end());
887 : if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset))
888 : return EC;
889 : }
890 :
891 114 : return sampleprof_error::success;
892 114 : }
893 :
894 : /// Read a GCC AutoFDO profile.
895 114 : ///
896 114 : /// This format is generated by the Linux Perf conversion tool at
897 114 : /// https://github.com/google/autofdo.
898 : std::error_code SampleProfileReaderGCC::read() {
899 : // Read the string table.
900 303 : if (std::error_code EC = readNameTable())
901 189 : return EC;
902 :
903 : // Read the source profile.
904 114 : if (std::error_code EC = readFunctionProfiles())
905 : return EC;
906 :
907 : return sampleprof_error::success;
908 : }
909 :
910 118 : bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
911 : StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart()));
912 4 : return Magic == "adcg*704";
913 0 : }
914 :
915 4 : std::error_code SampleProfileReaderItaniumRemapper::read() {
916 0 : // If the underlying data is in compact format, we can't remap it because
917 : // we don't know what the original function names were.
918 : if (getFormat() == SPF_Compact_Binary) {
919 4 : Ctx.diagnose(DiagnosticInfoSampleProfile(
920 0 : Buffer->getBufferIdentifier(),
921 4 : "Profile data remapping cannot be applied to profile data "
922 : "in compact format (original mangled names are not available).",
923 : DS_Warning));
924 4 : return sampleprof_error::success;
925 0 : }
926 :
927 4 : if (Error E = Remappings.read(*Buffer)) {
928 4 : handleAllErrors(
929 : std::move(E), [&](const SymbolRemappingParseError &ParseError) {
930 : reportError(ParseError.getLineNum(), ParseError.getMessage());
931 : });
932 : return sampleprof_error::malformed;
933 : }
934 :
935 50 : for (auto &Sample : getProfiles())
936 : if (auto Key = Remappings.insert(Sample.first()))
937 : SampleMap.insert({Key, &Sample.second});
938 :
939 : return sampleprof_error::success;
940 20 : }
941 0 :
942 : FunctionSamples *
943 20 : SampleProfileReaderItaniumRemapper::getSamplesFor(StringRef Fname) {
944 20 : if (auto Key = Remappings.lookup(Fname))
945 20 : return SampleMap.lookup(Key);
946 0 : return SampleProfileReader::getSamplesFor(Fname);
947 : }
948 :
949 30 : /// Prepare a memory buffer for the contents of \p Filename.
950 : ///
951 : /// \returns an error code indicating the status of the buffer.
952 : static ErrorOr<std::unique_ptr<MemoryBuffer>>
953 : setupMemoryBuffer(const Twine &Filename) {
954 : auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename);
955 : if (std::error_code EC = BufferOrErr.getError())
956 8 : return EC;
957 : auto Buffer = std::move(BufferOrErr.get());
958 8 :
959 0 : // Sanity check the file.
960 : if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint32_t>::max())
961 : return sampleprof_error::too_large;
962 8 :
963 0 : return std::move(Buffer);
964 : }
965 8 :
966 : /// Create a sample profile reader based on the format of the input file.
967 : ///
968 110 : /// \param Filename The file to open.
969 110 : ///
970 110 : /// \param C The LLVM context to use to emit diagnostics.
971 : ///
972 : /// \returns an error code indicating the status of the created reader.
973 4 : ErrorOr<std::unique_ptr<SampleProfileReader>>
974 : SampleProfileReader::create(const Twine &Filename, LLVMContext &C) {
975 : auto BufferOrError = setupMemoryBuffer(Filename);
976 4 : if (std::error_code EC = BufferOrError.getError())
977 0 : return EC;
978 0 : return create(BufferOrError.get(), C);
979 : }
980 :
981 : /// Create a sample profile remapper from the given input, to remap the
982 0 : /// function names in the given profile data.
983 : ///
984 : /// \param Filename The file to open.
985 12 : ///
986 0 : /// \param C The LLVM context to use to emit diagnostics.
987 : ///
988 : /// \param Underlying The underlying profile data reader to remap.
989 : ///
990 0 : /// \returns an error code indicating the status of the created reader.
991 : ErrorOr<std::unique_ptr<SampleProfileReader>>
992 : SampleProfileReaderItaniumRemapper::create(
993 14 : const Twine &Filename, LLVMContext &C,
994 6 : std::unique_ptr<SampleProfileReader> Underlying) {
995 6 : auto BufferOrError = setupMemoryBuffer(Filename);
996 : if (std::error_code EC = BufferOrError.getError())
997 4 : return EC;
998 : return llvm::make_unique<SampleProfileReaderItaniumRemapper>(
999 : std::move(BufferOrError.get()), C, std::move(Underlying));
1000 : }
1001 6 :
1002 6 : /// Create a sample profile reader based on the format of the input data.
1003 12 : ///
1004 0 : /// \param B The memory buffer to create the reader from (assumes ownership).
1005 : ///
1006 : /// \param C The LLVM context to use to emit diagnostics.
1007 : ///
1008 : /// \returns an error code indicating the status of the created reader.
1009 : ErrorOr<std::unique_ptr<SampleProfileReader>>
1010 : SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) {
1011 132 : std::unique_ptr<SampleProfileReader> Reader;
1012 132 : if (SampleProfileReaderRawBinary::hasFormat(*B))
1013 132 : Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));
1014 : else if (SampleProfileReaderCompactBinary::hasFormat(*B))
1015 : Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C));
1016 : else if (SampleProfileReaderGCC::hasFormat(*B))
1017 : Reader.reset(new SampleProfileReaderGCC(std::move(B), C));
1018 260 : else if (SampleProfileReaderText::hasFormat(*B))
1019 : Reader.reset(new SampleProfileReaderText(std::move(B), C));
1020 : else
1021 : return sampleprof_error::unrecognized_format;
1022 :
1023 : FunctionSamples::Format = Reader->getFormat();
1024 : if (std::error_code EC = Reader->readHeader())
1025 : return EC;
1026 :
1027 : return std::move(Reader);
1028 : }
1029 :
1030 : // For text and GCC file formats, we compute the summary after reading the
1031 : // profile. Binary format has the profile summary in its header.
1032 130 : void SampleProfileReader::computeSummary() {
1033 130 : SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1034 130 : for (const auto &I : Profiles) {
1035 : const FunctionSamples &Profile = I.second;
1036 128 : Builder.addRecord(Profile);
1037 : }
1038 : Summary = Builder.getSummary();
1039 : }
|
