Bug Summary

File:build/source/bolt/lib/Profile/DataAggregator.cpp
Warning:line 806, column 5
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name DataAggregator.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16 -I tools/bolt/lib/Profile -I /build/source/bolt/lib/Profile -I include -I /build/source/llvm/include -I /build/source/bolt/include -I tools/bolt/include -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1670191760 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-12-05-012027-15999-1 -x c++ /build/source/bolt/lib/Profile/DataAggregator.cpp
1//===- bolt/Profile/DataAggregator.cpp - Perf data aggregator -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions reads profile data written by perf record,
10// aggregate it and then write it back to an output file.
11//
12//===----------------------------------------------------------------------===//
13
14#include "bolt/Profile/DataAggregator.h"
15#include "bolt/Core/BinaryContext.h"
16#include "bolt/Core/BinaryFunction.h"
17#include "bolt/Profile/BoltAddressTranslation.h"
18#include "bolt/Profile/Heatmap.h"
19#include "bolt/Utils/CommandLineOpts.h"
20#include "bolt/Utils/Utils.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/ScopeExit.h"
23#include "llvm/Support/CommandLine.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/Errc.h"
26#include "llvm/Support/FileSystem.h"
27#include "llvm/Support/Process.h"
28#include "llvm/Support/Program.h"
29#include "llvm/Support/Regex.h"
30#include "llvm/Support/Timer.h"
31#include "llvm/Support/raw_ostream.h"
32#include <map>
33#include <unordered_map>
34#include <utility>
35
36#define DEBUG_TYPE"aggregator" "aggregator"
37
38using namespace llvm;
39using namespace bolt;
40
41namespace opts {
42
43static cl::opt<bool>
44 BasicAggregation("nl",
45 cl::desc("aggregate basic samples (without LBR info)"),
46 cl::cat(AggregatorCategory));
47
48static cl::opt<bool>
49FilterMemProfile("filter-mem-profile",
50 cl::desc("if processing a memory profile, filter out stack or heap accesses "
51 "that won't be useful for BOLT to reduce profile file size"),
52 cl::init(true),
53 cl::cat(AggregatorCategory));
54
55static cl::opt<unsigned long long>
56FilterPID("pid",
57 cl::desc("only use samples from process with specified PID"),
58 cl::init(0),
59 cl::Optional,
60 cl::cat(AggregatorCategory));
61
62static cl::opt<bool>
63IgnoreBuildID("ignore-build-id",
64 cl::desc("continue even if build-ids in input binary and perf.data mismatch"),
65 cl::init(false),
66 cl::cat(AggregatorCategory));
67
68static cl::opt<bool> IgnoreInterruptLBR(
69 "ignore-interrupt-lbr",
70 cl::desc("ignore kernel interrupt LBR that happens asynchronously"),
71 cl::init(true), cl::cat(AggregatorCategory));
72
73static cl::opt<unsigned long long>
74MaxSamples("max-samples",
75 cl::init(-1ULL),
76 cl::desc("maximum number of samples to read from LBR profile"),
77 cl::Optional,
78 cl::Hidden,
79 cl::cat(AggregatorCategory));
80
81extern cl::opt<opts::ProfileFormatKind> ProfileFormat;
82
83cl::opt<bool> ReadPreAggregated(
84 "pa", cl::desc("skip perf and read data from a pre-aggregated file format"),
85 cl::cat(AggregatorCategory));
86
87static cl::opt<bool>
88TimeAggregator("time-aggr",
89 cl::desc("time BOLT aggregator"),
90 cl::init(false),
91 cl::ZeroOrMore,
92 cl::cat(AggregatorCategory));
93
94static cl::opt<bool>
95 UseEventPC("use-event-pc",
96 cl::desc("use event PC in combination with LBR sampling"),
97 cl::cat(AggregatorCategory));
98
99static cl::opt<bool> WriteAutoFDOData(
100 "autofdo", cl::desc("generate autofdo textual data instead of bolt data"),
101 cl::cat(AggregatorCategory));
102
103} // namespace opts
104
105namespace {
106
107const char TimerGroupName[] = "aggregator";
108const char TimerGroupDesc[] = "Aggregator";
109
110std::vector<SectionNameAndRange> getTextSections(const BinaryContext *BC) {
111 std::vector<SectionNameAndRange> sections;
112 for (BinarySection &Section : BC->sections()) {
113 if (!Section.isText())
114 continue;
115 if (Section.getSize() == 0)
116 continue;
117 sections.push_back(
118 {Section.getName(), Section.getAddress(), Section.getEndAddress()});
119 }
120 llvm::sort(sections,
121 [](const SectionNameAndRange &A, const SectionNameAndRange &B) {
122 return A.BeginAddress < B.BeginAddress;
123 });
124 return sections;
125}
126}
127
128constexpr uint64_t DataAggregator::KernelBaseAddr;
129
130DataAggregator::~DataAggregator() { deleteTempFiles(); }
131
132namespace {
133void deleteTempFile(const std::string &FileName) {
134 if (std::error_code Errc = sys::fs::remove(FileName.c_str()))
135 errs() << "PERF2BOLT: failed to delete temporary file " << FileName
136 << " with error " << Errc.message() << "\n";
137}
138}
139
140void DataAggregator::deleteTempFiles() {
141 for (std::string &FileName : TempFiles)
142 deleteTempFile(FileName);
143 TempFiles.clear();
144}
145
146void DataAggregator::findPerfExecutable() {
147 Optional<std::string> PerfExecutable =
148 sys::Process::FindInEnvPath("PATH", "perf");
149 if (!PerfExecutable) {
150 outs() << "PERF2BOLT: No perf executable found!\n";
151 exit(1);
152 }
153 PerfPath = *PerfExecutable;
154}
155
156void DataAggregator::start() {
157 outs() << "PERF2BOLT: Starting data aggregation job for " << Filename << "\n";
158
159 // Don't launch perf for pre-aggregated files
160 if (opts::ReadPreAggregated)
161 return;
162
163 findPerfExecutable();
164
165 if (opts::BasicAggregation)
166 launchPerfProcess("events without LBR",
167 MainEventsPPI,
168 "script -F pid,event,ip",
169 /*Wait = */false);
170 else
171 launchPerfProcess("branch events",
172 MainEventsPPI,
173 "script -F pid,ip,brstack",
174 /*Wait = */false);
175
176 // Note: we launch script for mem events regardless of the option, as the
177 // command fails fairly fast if mem events were not collected.
178 launchPerfProcess("mem events",
179 MemEventsPPI,
180 "script -F pid,event,addr,ip",
181 /*Wait = */false);
182
183 launchPerfProcess("process events",
184 MMapEventsPPI,
185 "script --show-mmap-events",
186 /*Wait = */false);
187
188 launchPerfProcess("task events",
189 TaskEventsPPI,
190 "script --show-task-events",
191 /*Wait = */false);
192}
193
194void DataAggregator::abort() {
195 if (opts::ReadPreAggregated)
196 return;
197
198 std::string Error;
199
200 // Kill subprocesses in case they are not finished
201 sys::Wait(TaskEventsPPI.PI, 1, false, &Error);
202 sys::Wait(MMapEventsPPI.PI, 1, false, &Error);
203 sys::Wait(MainEventsPPI.PI, 1, false, &Error);
204 sys::Wait(MemEventsPPI.PI, 1, false, &Error);
205
206 deleteTempFiles();
207
208 exit(1);
209}
210
211void DataAggregator::launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,
212 const char *ArgsString, bool Wait) {
213 SmallVector<StringRef, 4> Argv;
214
215 outs() << "PERF2BOLT: spawning perf job to read " << Name << '\n';
216 Argv.push_back(PerfPath.data());
217
218 char *WritableArgsString = strdup(ArgsString);
219 char *Str = WritableArgsString;
220 do {
221 Argv.push_back(Str);
222 while (*Str && *Str != ' ')
223 ++Str;
224 if (!*Str)
225 break;
226 *Str++ = 0;
227 } while (true);
228
229 Argv.push_back("-f");
230 Argv.push_back("-i");
231 Argv.push_back(Filename.c_str());
232
233 if (std::error_code Errc =
234 sys::fs::createTemporaryFile("perf.script", "out", PPI.StdoutPath)) {
235 errs() << "PERF2BOLT: failed to create temporary file " << PPI.StdoutPath
236 << " with error " << Errc.message() << "\n";
237 exit(1);
238 }
239 TempFiles.push_back(PPI.StdoutPath.data());
240
241 if (std::error_code Errc =
242 sys::fs::createTemporaryFile("perf.script", "err", PPI.StderrPath)) {
243 errs() << "PERF2BOLT: failed to create temporary file " << PPI.StderrPath
244 << " with error " << Errc.message() << "\n";
245 exit(1);
246 }
247 TempFiles.push_back(PPI.StderrPath.data());
248
249 std::optional<StringRef> Redirects[] = {
250 std::nullopt, // Stdin
251 StringRef(PPI.StdoutPath.data()), // Stdout
252 StringRef(PPI.StderrPath.data())}; // Stderr
253
254 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "Launching perf: "; for (
StringRef Arg : Argv) dbgs() << Arg << " "; dbgs(
) << " 1> " << PPI.StdoutPath.data() << " 2> "
<< PPI.StderrPath.data() << "\n"; }; } } while (
false)
255 dbgs() << "Launching perf: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "Launching perf: "; for (
StringRef Arg : Argv) dbgs() << Arg << " "; dbgs(
) << " 1> " << PPI.StdoutPath.data() << " 2> "
<< PPI.StderrPath.data() << "\n"; }; } } while (
false)
256 for (StringRef Arg : Argv)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "Launching perf: "; for (
StringRef Arg : Argv) dbgs() << Arg << " "; dbgs(
) << " 1> " << PPI.StdoutPath.data() << " 2> "
<< PPI.StderrPath.data() << "\n"; }; } } while (
false)
257 dbgs() << Arg << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "Launching perf: "; for (
StringRef Arg : Argv) dbgs() << Arg << " "; dbgs(
) << " 1> " << PPI.StdoutPath.data() << " 2> "
<< PPI.StderrPath.data() << "\n"; }; } } while (
false)
258 dbgs() << " 1> " << PPI.StdoutPath.data() << " 2> " << PPI.StderrPath.data()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "Launching perf: "; for (
StringRef Arg : Argv) dbgs() << Arg << " "; dbgs(
) << " 1> " << PPI.StdoutPath.data() << " 2> "
<< PPI.StderrPath.data() << "\n"; }; } } while (
false)
259 << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "Launching perf: "; for (
StringRef Arg : Argv) dbgs() << Arg << " "; dbgs(
) << " 1> " << PPI.StdoutPath.data() << " 2> "
<< PPI.StderrPath.data() << "\n"; }; } } while (
false)
260 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "Launching perf: "; for (
StringRef Arg : Argv) dbgs() << Arg << " "; dbgs(
) << " 1> " << PPI.StdoutPath.data() << " 2> "
<< PPI.StderrPath.data() << "\n"; }; } } while (
false)
;
261
262 if (Wait)
263 PPI.PI.ReturnCode = sys::ExecuteAndWait(PerfPath.data(), Argv,
264 /*envp*/ std::nullopt, Redirects);
265 else
266 PPI.PI = sys::ExecuteNoWait(PerfPath.data(), Argv, /*envp*/ std::nullopt,
267 Redirects);
268
269 free(WritableArgsString);
270}
271
272void DataAggregator::processFileBuildID(StringRef FileBuildID) {
273 PerfProcessInfo BuildIDProcessInfo;
274 launchPerfProcess("buildid list",
275 BuildIDProcessInfo,
276 "buildid-list",
277 /*Wait = */true);
278
279 if (BuildIDProcessInfo.PI.ReturnCode != 0) {
280 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
281 MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StderrPath.data());
282 StringRef ErrBuf = (*MB)->getBuffer();
283
284 errs() << "PERF-ERROR: return code " << BuildIDProcessInfo.PI.ReturnCode
285 << '\n';
286 errs() << ErrBuf;
287 return;
288 }
289
290 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
291 MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StdoutPath.data());
292 if (std::error_code EC = MB.getError()) {
293 errs() << "Cannot open " << BuildIDProcessInfo.StdoutPath.data() << ": "
294 << EC.message() << "\n";
295 return;
296 }
297
298 FileBuf = std::move(*MB);
299 ParsingBuf = FileBuf->getBuffer();
300
301 Optional<StringRef> FileName = getFileNameForBuildID(FileBuildID);
302 if (!FileName) {
303 if (hasAllBuildIDs()) {
304 errs() << "PERF2BOLT-ERROR: failed to match build-id from perf output. "
305 "This indicates the input binary supplied for data aggregation "
306 "is not the same recorded by perf when collecting profiling "
307 "data, or there were no samples recorded for the binary. "
308 "Use -ignore-build-id option to override.\n";
309 if (!opts::IgnoreBuildID)
310 abort();
311 } else {
312 errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "
313 "data was recorded without it\n";
314 return;
315 }
316 } else if (*FileName != llvm::sys::path::filename(BC->getFilename())) {
317 errs() << "PERF2BOLT-WARNING: build-id matched a different file name\n";
318 BuildIDBinaryName = std::string(*FileName);
319 } else {
320 outs() << "PERF2BOLT: matched build-id and file name\n";
321 }
322}
323
324bool DataAggregator::checkPerfDataMagic(StringRef FileName) {
325 if (opts::ReadPreAggregated)
326 return true;
327
328 Expected<sys::fs::file_t> FD = sys::fs::openNativeFileForRead(FileName);
329 if (!FD)
330 return false;
331
332 char Buf[7] = {0, 0, 0, 0, 0, 0, 0};
333
334 auto Close = make_scope_exit([&] { sys::fs::closeFile(*FD); });
335 Expected<size_t> BytesRead = sys::fs::readNativeFileSlice(
336 *FD, makeMutableArrayRef(Buf, sizeof(Buf)), 0);
337 if (!BytesRead || *BytesRead != 7)
338 return false;
339
340 if (strncmp(Buf, "PERFILE", 7) == 0)
341 return true;
342 return false;
343}
344
345void DataAggregator::parsePreAggregated() {
346 std::string Error;
347
348 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
349 MemoryBuffer::getFileOrSTDIN(Filename);
350 if (std::error_code EC = MB.getError()) {
351 errs() << "PERF2BOLT-ERROR: cannot open " << Filename << ": "
352 << EC.message() << "\n";
353 exit(1);
354 }
355
356 FileBuf = std::move(*MB);
357 ParsingBuf = FileBuf->getBuffer();
358 Col = 0;
359 Line = 1;
360 if (parsePreAggregatedLBRSamples()) {
361 errs() << "PERF2BOLT: failed to parse samples\n";
362 exit(1);
363 }
364}
365
366std::error_code DataAggregator::writeAutoFDOData(StringRef OutputFilename) {
367 outs() << "PERF2BOLT: writing data for autofdo tools...\n";
368 NamedRegionTimer T("writeAutoFDO", "Processing branch events", TimerGroupName,
369 TimerGroupDesc, opts::TimeAggregator);
370
371 std::error_code EC;
372 raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None);
373 if (EC)
374 return EC;
375
376 // Format:
377 // number of unique traces
378 // from_1-to_1:count_1
379 // from_2-to_2:count_2
380 // ......
381 // from_n-to_n:count_n
382 // number of unique sample addresses
383 // addr_1:count_1
384 // addr_2:count_2
385 // ......
386 // addr_n:count_n
387 // number of unique LBR entries
388 // src_1->dst_1:count_1
389 // src_2->dst_2:count_2
390 // ......
391 // src_n->dst_n:count_n
392
393 const uint64_t FirstAllocAddress = this->BC->FirstAllocAddress;
394
395 // AutoFDO addresses are relative to the first allocated loadable program
396 // segment
397 auto filterAddress = [&FirstAllocAddress](uint64_t Address) -> uint64_t {
398 if (Address < FirstAllocAddress)
399 return 0;
400 return Address - FirstAllocAddress;
401 };
402
403 OutFile << FallthroughLBRs.size() << "\n";
404 for (const auto &AggrLBR : FallthroughLBRs) {
405 const Trace &Trace = AggrLBR.first;
406 const FTInfo &Info = AggrLBR.second;
407 OutFile << Twine::utohexstr(filterAddress(Trace.From)) << "-"
408 << Twine::utohexstr(filterAddress(Trace.To)) << ":"
409 << (Info.InternCount + Info.ExternCount) << "\n";
410 }
411
412 OutFile << BasicSamples.size() << "\n";
413 for (const auto &Sample : BasicSamples) {
414 uint64_t PC = Sample.first;
415 uint64_t HitCount = Sample.second;
416 OutFile << Twine::utohexstr(filterAddress(PC)) << ":" << HitCount << "\n";
417 }
418
419 OutFile << BranchLBRs.size() << "\n";
420 for (const auto &AggrLBR : BranchLBRs) {
421 const Trace &Trace = AggrLBR.first;
422 const BranchInfo &Info = AggrLBR.second;
423 OutFile << Twine::utohexstr(filterAddress(Trace.From)) << "->"
424 << Twine::utohexstr(filterAddress(Trace.To)) << ":"
425 << Info.TakenCount << "\n";
426 }
427
428 outs() << "PERF2BOLT: wrote " << FallthroughLBRs.size() << " unique traces, "
429 << BasicSamples.size() << " sample addresses and " << BranchLBRs.size()
430 << " unique branches to " << OutputFilename << "\n";
431
432 return std::error_code();
433}
434
435void DataAggregator::filterBinaryMMapInfo() {
436 if (opts::FilterPID) {
437 auto MMapInfoIter = BinaryMMapInfo.find(opts::FilterPID);
438 if (MMapInfoIter != BinaryMMapInfo.end()) {
439 MMapInfo MMap = MMapInfoIter->second;
440 BinaryMMapInfo.clear();
441 BinaryMMapInfo.insert(std::make_pair(MMap.PID, MMap));
442 } else {
443 if (errs().has_colors())
444 errs().changeColor(raw_ostream::RED);
445 errs() << "PERF2BOLT-ERROR: could not find a profile matching PID \""
446 << opts::FilterPID << "\""
447 << " for binary \"" << BC->getFilename() << "\".";
448 assert(!BinaryMMapInfo.empty() && "No memory map for matching binary")(static_cast <bool> (!BinaryMMapInfo.empty() &&
"No memory map for matching binary") ? void (0) : __assert_fail
("!BinaryMMapInfo.empty() && \"No memory map for matching binary\""
, "bolt/lib/Profile/DataAggregator.cpp", 448, __extension__ __PRETTY_FUNCTION__
))
;
449 errs() << " Profile for the following process is available:\n";
450 for (std::pair<const uint64_t, MMapInfo> &MMI : BinaryMMapInfo)
451 outs() << " " << MMI.second.PID
452 << (MMI.second.Forked ? " (forked)\n" : "\n");
453
454 if (errs().has_colors())
455 errs().resetColor();
456
457 exit(1);
458 }
459 }
460}
461
462Error DataAggregator::preprocessProfile(BinaryContext &BC) {
463 this->BC = &BC;
464
465 if (opts::ReadPreAggregated) {
466 parsePreAggregated();
467 return Error::success();
468 }
469
470 if (Optional<StringRef> FileBuildID = BC.getFileBuildID()) {
471 outs() << "BOLT-INFO: binary build-id is: " << *FileBuildID << "\n";
472 processFileBuildID(*FileBuildID);
473 } else {
474 errs() << "BOLT-WARNING: build-id will not be checked because we could "
475 "not read one from input binary\n";
476 }
477
478 auto prepareToParse = [&](StringRef Name, PerfProcessInfo &Process) {
479 std::string Error;
480 outs() << "PERF2BOLT: waiting for perf " << Name
481 << " collection to finish...\n";
482 sys::ProcessInfo PI = sys::Wait(Process.PI, 0, true, &Error);
483
484 if (!Error.empty()) {
485 errs() << "PERF-ERROR: " << PerfPath << ": " << Error << "\n";
486 deleteTempFiles();
487 exit(1);
488 }
489
490 if (PI.ReturnCode != 0) {
491 ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorMB =
492 MemoryBuffer::getFileOrSTDIN(Process.StderrPath.data());
493 StringRef ErrBuf = (*ErrorMB)->getBuffer();
494
495 errs() << "PERF-ERROR: return code " << PI.ReturnCode << "\n";
496 errs() << ErrBuf;
497 deleteTempFiles();
498 exit(1);
499 }
500
501 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
502 MemoryBuffer::getFileOrSTDIN(Process.StdoutPath.data());
503 if (std::error_code EC = MB.getError()) {
504 errs() << "Cannot open " << Process.StdoutPath.data() << ": "
505 << EC.message() << "\n";
506 deleteTempFiles();
507 exit(1);
508 }
509
510 FileBuf = std::move(*MB);
511 ParsingBuf = FileBuf->getBuffer();
512 Col = 0;
513 Line = 1;
514 };
515
516 if (opts::LinuxKernelMode) {
517 // Current MMap parsing logic does not work with linux kernel.
518 // MMap entries for linux kernel uses PERF_RECORD_MMAP
519 // format instead of typical PERF_RECORD_MMAP2 format.
520 // Since linux kernel address mapping is absolute (same as
521 // in the ELF file), we avoid parsing MMap in linux kernel mode.
522 // While generating optimized linux kernel binary, we may need
523 // to parse MMap entries.
524
525 // In linux kernel mode, we analyze and optimize
526 // all linux kernel binary instructions, irrespective
527 // of whether they are due to system calls or due to
528 // interrupts. Therefore, we cannot ignore interrupt
529 // in Linux kernel mode.
530 opts::IgnoreInterruptLBR = false;
531 } else {
532 prepareToParse("mmap events", MMapEventsPPI);
533 if (parseMMapEvents())
534 errs() << "PERF2BOLT: failed to parse mmap events\n";
535 }
536
537 prepareToParse("task events", TaskEventsPPI);
538 if (parseTaskEvents())
539 errs() << "PERF2BOLT: failed to parse task events\n";
540
541 filterBinaryMMapInfo();
542 prepareToParse("events", MainEventsPPI);
543
544 if (opts::HeatmapMode) {
545 if (std::error_code EC = printLBRHeatMap()) {
546 errs() << "ERROR: failed to print heat map: " << EC.message() << '\n';
547 exit(1);
548 }
549 exit(0);
550 }
551
552 if ((!opts::BasicAggregation && parseBranchEvents()) ||
553 (opts::BasicAggregation && parseBasicEvents()))
554 errs() << "PERF2BOLT: failed to parse samples\n";
555
556 // We can finish early if the goal is just to generate data for autofdo
557 if (opts::WriteAutoFDOData) {
558 if (std::error_code EC = writeAutoFDOData(opts::OutputFilename))
559 errs() << "Error writing autofdo data to file: " << EC.message() << "\n";
560
561 deleteTempFiles();
562 exit(0);
563 }
564
565 // Special handling for memory events
566 std::string Error;
567 sys::ProcessInfo PI = sys::Wait(MemEventsPPI.PI, 0, true, &Error);
568 if (PI.ReturnCode != 0) {
569 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
570 MemoryBuffer::getFileOrSTDIN(MemEventsPPI.StderrPath.data());
571 StringRef ErrBuf = (*MB)->getBuffer();
572
573 deleteTempFiles();
574
575 Regex NoData("Samples for '.*' event do not have ADDR attribute set. "
576 "Cannot print 'addr' field.");
577 if (!NoData.match(ErrBuf)) {
578 errs() << "PERF-ERROR: return code " << PI.ReturnCode << "\n";
579 errs() << ErrBuf;
580 exit(1);
581 }
582 return Error::success();
583 }
584
585 ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
586 MemoryBuffer::getFileOrSTDIN(MemEventsPPI.StdoutPath.data());
587 if (std::error_code EC = MB.getError()) {
588 errs() << "Cannot open " << MemEventsPPI.StdoutPath.data() << ": "
589 << EC.message() << "\n";
590 deleteTempFiles();
591 exit(1);
592 }
593
594 FileBuf = std::move(*MB);
595 ParsingBuf = FileBuf->getBuffer();
596 Col = 0;
597 Line = 1;
598 if (const std::error_code EC = parseMemEvents())
599 errs() << "PERF2BOLT: failed to parse memory events: " << EC.message()
600 << '\n';
601
602 deleteTempFiles();
603
604 return Error::success();
605}
606
607Error DataAggregator::readProfile(BinaryContext &BC) {
608 processProfile(BC);
1
Calling 'DataAggregator::processProfile'
609
610 for (auto &BFI : BC.getBinaryFunctions()) {
611 BinaryFunction &Function = BFI.second;
612 convertBranchData(Function);
613 }
614
615 if (opts::AggregateOnly &&
616 opts::ProfileFormat == opts::ProfileFormatKind::PF_Fdata) {
617 if (std::error_code EC = writeAggregatedFile(opts::OutputFilename))
618 report_error("cannot create output data file", EC);
619 }
620
621 return Error::success();
622}
623
624bool DataAggregator::mayHaveProfileData(const BinaryFunction &Function) {
625 return Function.hasProfileAvailable();
626}
627
628void DataAggregator::processProfile(BinaryContext &BC) {
629 if (opts::ReadPreAggregated)
2
Assuming the condition is true
3
Taking true branch
630 processPreAggregated();
4
Calling 'DataAggregator::processPreAggregated'
631 else if (opts::BasicAggregation)
632 processBasicEvents();
633 else
634 processBranchEvents();
635
636 processMemEvents();
637
638 // Mark all functions with registered events as having a valid profile.
639 for (auto &BFI : BC.getBinaryFunctions()) {
640 BinaryFunction &BF = BFI.second;
641 if (getBranchData(BF)) {
642 const auto Flags = opts::BasicAggregation ? BinaryFunction::PF_SAMPLE
643 : BinaryFunction::PF_LBR;
644 BF.markProfiled(Flags);
645 }
646 }
647
648 // Release intermediate storage.
649 clear(BranchLBRs);
650 clear(FallthroughLBRs);
651 clear(AggregatedLBRs);
652 clear(BasicSamples);
653 clear(MemSamples);
654}
655
656BinaryFunction *
657DataAggregator::getBinaryFunctionContainingAddress(uint64_t Address) const {
658 if (!BC->containsAddress(Address))
9
Assuming the condition is false
10
Taking false branch
16
Value assigned to 'DebugFlag', which participates in a condition later
17
Assuming the condition is true
18
Taking true branch
659 return nullptr;
660
661 return BC->getBinaryFunctionContainingAddress(Address, /*CheckPastEnd=*/false,
11
Returning pointer, which participates in a condition later
12
Returning pointer
662 /*UseMaxSize=*/true);
663}
664
665StringRef DataAggregator::getLocationName(BinaryFunction &Func,
666 uint64_t Count) {
667 if (!BAT)
668 return Func.getOneName();
669
670 const BinaryFunction *OrigFunc = &Func;
671 if (const uint64_t HotAddr = BAT->fetchParentAddress(Func.getAddress())) {
672 NumColdSamples += Count;
673 BinaryFunction *HotFunc = getBinaryFunctionContainingAddress(HotAddr);
674 if (HotFunc)
675 OrigFunc = HotFunc;
676 }
677 // If it is a local function, prefer the name containing the file name where
678 // the local function was declared
679 for (StringRef AlternativeName : OrigFunc->getNames()) {
680 size_t FileNameIdx = AlternativeName.find('/');
681 // Confirm the alternative name has the pattern Symbol/FileName/1 before
682 // using it
683 if (FileNameIdx == StringRef::npos ||
684 AlternativeName.find('/', FileNameIdx + 1) == StringRef::npos)
685 continue;
686 return AlternativeName;
687 }
688 return OrigFunc->getOneName();
689}
690
691bool DataAggregator::doSample(BinaryFunction &Func, uint64_t Address,
692 uint64_t Count) {
693 auto I = NamesToSamples.find(Func.getOneName());
694 if (I == NamesToSamples.end()) {
695 bool Success;
696 StringRef LocName = getLocationName(Func, Count);
697 std::tie(I, Success) = NamesToSamples.insert(
698 std::make_pair(Func.getOneName(),
699 FuncSampleData(LocName, FuncSampleData::ContainerTy())));
700 }
701
702 Address -= Func.getAddress();
703 if (BAT)
704 Address = BAT->translate(Func.getAddress(), Address, /*IsBranchSrc=*/false);
705
706 I->second.bumpCount(Address, Count);
707 return true;
708}
709
710bool DataAggregator::doIntraBranch(BinaryFunction &Func, uint64_t From,
711 uint64_t To, uint64_t Count,
712 uint64_t Mispreds) {
713 FuncBranchData *AggrData = getBranchData(Func);
714 if (!AggrData) {
715 AggrData = &NamesToBranches[Func.getOneName()];
716 AggrData->Name = getLocationName(Func, Count);
717 setBranchData(Func, AggrData);
718 }
719
720 From -= Func.getAddress();
721 To -= Func.getAddress();
722 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: bumpBranchCount: " << Func.getPrintName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
723 << " @ " << Twine::utohexstr(From) << " -> "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
724 << Func.getPrintName() << " @ " << Twine::utohexstr(To)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
725 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
;
726 if (BAT) {
727 From = BAT->translate(Func.getAddress(), From, /*IsBranchSrc=*/true);
728 To = BAT->translate(Func.getAddress(), To, /*IsBranchSrc=*/false);
729 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: BAT translation on bumpBranchCount: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: BAT translation on bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
730 << Func.getPrintName() << " @ " << Twine::utohexstr(From)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: BAT translation on bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
731 << " -> " << Func.getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: BAT translation on bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
732 << Twine::utohexstr(To) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "BOLT-DEBUG: BAT translation on bumpBranchCount: "
<< Func.getPrintName() << " @ " << Twine::
utohexstr(From) << " -> " << Func.getPrintName
() << " @ " << Twine::utohexstr(To) << '\n'
; } } while (false)
;
733 }
734
735 AggrData->bumpBranchCount(From, To, Count, Mispreds);
736 return true;
737}
738
739bool DataAggregator::doInterBranch(BinaryFunction *FromFunc,
740 BinaryFunction *ToFunc, uint64_t From,
741 uint64_t To, uint64_t Count,
742 uint64_t Mispreds) {
743 FuncBranchData *FromAggrData = nullptr;
744 FuncBranchData *ToAggrData = nullptr;
745 StringRef SrcFunc;
746 StringRef DstFunc;
747 if (FromFunc) {
748 SrcFunc = getLocationName(*FromFunc, Count);
749 FromAggrData = getBranchData(*FromFunc);
750 if (!FromAggrData) {
751 FromAggrData = &NamesToBranches[FromFunc->getOneName()];
752 FromAggrData->Name = SrcFunc;
753 setBranchData(*FromFunc, FromAggrData);
754 }
755 From -= FromFunc->getAddress();
756 if (BAT)
757 From = BAT->translate(FromFunc->getAddress(), From, /*IsBranchSrc=*/true);
758
759 recordExit(*FromFunc, From, Mispreds, Count);
760 }
761 if (ToFunc) {
762 DstFunc = getLocationName(*ToFunc, 0);
763 ToAggrData = getBranchData(*ToFunc);
764 if (!ToAggrData) {
765 ToAggrData = &NamesToBranches[ToFunc->getOneName()];
766 ToAggrData->Name = DstFunc;
767 setBranchData(*ToFunc, ToAggrData);
768 }
769 To -= ToFunc->getAddress();
770 if (BAT)
771 To = BAT->translate(ToFunc->getAddress(), To, /*IsBranchSrc=*/false);
772
773 recordEntry(*ToFunc, To, Mispreds, Count);
774 }
775
776 if (FromAggrData)
777 FromAggrData->bumpCallCount(From, Location(!DstFunc.empty(), DstFunc, To),
778 Count, Mispreds);
779 if (ToAggrData)
780 ToAggrData->bumpEntryCount(Location(!SrcFunc.empty(), SrcFunc, From), To,
781 Count, Mispreds);
782 return true;
783}
784
785bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
786 uint64_t Mispreds) {
787 BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(From);
788 BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(To);
789 if (!FromFunc && !ToFunc)
790 return false;
791
792 if (FromFunc == ToFunc) {
793 recordBranch(*FromFunc, From - FromFunc->getAddress(),
794 To - FromFunc->getAddress(), Count, Mispreds);
795 return doIntraBranch(*FromFunc, From, To, Count, Mispreds);
796 }
797
798 return doInterBranch(FromFunc, ToFunc, From, To, Count, Mispreds);
799}
800
801bool DataAggregator::doTrace(const LBREntry &First, const LBREntry &Second,
802 uint64_t Count) {
803 BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(First.To);
8
Calling 'DataAggregator::getBinaryFunctionContainingAddress'
13
Returning from 'DataAggregator::getBinaryFunctionContainingAddress'
14
'FromFunc' initialized here
804 BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(Second.From);
15
Calling 'DataAggregator::getBinaryFunctionContainingAddress'
19
Returning from 'DataAggregator::getBinaryFunctionContainingAddress'
805 if (!FromFunc || !ToFunc) {
20
Assuming 'FromFunc' is null
21
Assuming pointer value is null
806 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
22
Assuming 'DebugFlag' is true
23
Assuming the condition is true
24
Taking true branch
25
Called C++ object pointer is null
807 dbgs() << "Out of range trace starting in " << FromFunc->getPrintName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
808 << " @ " << Twine::utohexstr(First.To - FromFunc->getAddress())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
809 << " and ending in " << ToFunc->getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
810 << ToFunc->getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
811 << Twine::utohexstr(Second.From - ToFunc->getAddress()) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
;
812 NumLongRangeTraces += Count;
813 return false;
814 }
815 if (FromFunc != ToFunc) {
816 NumInvalidTraces += Count;
817 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
818 dbgs() << "Invalid trace starting in " << FromFunc->getPrintName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
819 << " @ " << Twine::utohexstr(First.To - FromFunc->getAddress())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
820 << " and ending in " << ToFunc->getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
821 << ToFunc->getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
822 << Twine::utohexstr(Second.From - ToFunc->getAddress()) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
;
823 return false;
824 }
825
826 Optional<BoltAddressTranslation::FallthroughListTy> FTs =
827 BAT ? BAT->getFallthroughsInTrace(FromFunc->getAddress(), First.To,
828 Second.From)
829 : getFallthroughsInTrace(*FromFunc, First, Second, Count);
830 if (!FTs) {
831 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
832 dbgs() << "Invalid trace starting in " << FromFunc->getPrintName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
833 << " @ " << Twine::utohexstr(First.To - FromFunc->getAddress())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
834 << " and ending in " << ToFunc->getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
835 << ToFunc->getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
836 << Twine::utohexstr(Second.From - ToFunc->getAddress()) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< FromFunc->getPrintName() << " @ " <<
Twine::utohexstr(First.To - FromFunc->getAddress()) <<
" and ending in " << ToFunc->getPrintName() <<
" @ " << ToFunc->getPrintName() << " @ " <<
Twine::utohexstr(Second.From - ToFunc->getAddress()) <<
'\n'; } } while (false)
;
837 NumInvalidTraces += Count;
838 return false;
839 }
840
841 LLVM_DEBUG(dbgs() << "Processing " << FTs->size() << " fallthroughs for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Processing " << FTs->
size() << " fallthroughs for " << FromFunc->getPrintName
() << ":" << Twine::utohexstr(First.To) << " to "
<< Twine::utohexstr(Second.From) << ".\n"; } } while
(false)
842 << FromFunc->getPrintName() << ":"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Processing " << FTs->
size() << " fallthroughs for " << FromFunc->getPrintName
() << ":" << Twine::utohexstr(First.To) << " to "
<< Twine::utohexstr(Second.From) << ".\n"; } } while
(false)
843 << Twine::utohexstr(First.To) << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Processing " << FTs->
size() << " fallthroughs for " << FromFunc->getPrintName
() << ":" << Twine::utohexstr(First.To) << " to "
<< Twine::utohexstr(Second.From) << ".\n"; } } while
(false)
844 << Twine::utohexstr(Second.From) << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Processing " << FTs->
size() << " fallthroughs for " << FromFunc->getPrintName
() << ":" << Twine::utohexstr(First.To) << " to "
<< Twine::utohexstr(Second.From) << ".\n"; } } while
(false)
;
845 for (const std::pair<uint64_t, uint64_t> &Pair : *FTs)
846 doIntraBranch(*FromFunc, Pair.first + FromFunc->getAddress(),
847 Pair.second + FromFunc->getAddress(), Count, false);
848
849 return true;
850}
851
852bool DataAggregator::recordTrace(
853 BinaryFunction &BF,
854 const LBREntry &FirstLBR,
855 const LBREntry &SecondLBR,
856 uint64_t Count,
857 SmallVector<std::pair<uint64_t, uint64_t>, 16> *Branches) const {
858 BinaryContext &BC = BF.getBinaryContext();
859
860 if (!BF.isSimple())
861 return false;
862
863 assert(BF.hasCFG() && "can only record traces in CFG state")(static_cast <bool> (BF.hasCFG() && "can only record traces in CFG state"
) ? void (0) : __assert_fail ("BF.hasCFG() && \"can only record traces in CFG state\""
, "bolt/lib/Profile/DataAggregator.cpp", 863, __extension__ __PRETTY_FUNCTION__
))
;
864
865 // Offsets of the trace within this function.
866 const uint64_t From = FirstLBR.To - BF.getAddress();
867 const uint64_t To = SecondLBR.From - BF.getAddress();
868
869 if (From > To)
870 return false;
871
872 const BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(From);
873 const BinaryBasicBlock *ToBB = BF.getBasicBlockContainingOffset(To);
874
875 if (!FromBB || !ToBB)
876 return false;
877
878 // Adjust FromBB if the first LBR is a return from the last instruction in
879 // the previous block (that instruction should be a call).
880 if (From == FromBB->getOffset() && !BF.containsAddress(FirstLBR.From) &&
881 !FromBB->isEntryPoint() && !FromBB->isLandingPad()) {
882 const BinaryBasicBlock *PrevBB =
883 BF.getLayout().getBlock(FromBB->getIndex() - 1);
884 if (PrevBB->getSuccessor(FromBB->getLabel())) {
885 const MCInst *Instr = PrevBB->getLastNonPseudoInstr();
886 if (Instr && BC.MIB->isCall(*Instr))
887 FromBB = PrevBB;
888 else
889 LLVM_DEBUG(dbgs() << "invalid incoming LBR (no call): " << FirstLBRdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "invalid incoming LBR (no call): "
<< FirstLBR << '\n'; } } while (false)
890 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "invalid incoming LBR (no call): "
<< FirstLBR << '\n'; } } while (false)
;
891 } else {
892 LLVM_DEBUG(dbgs() << "invalid incoming LBR: " << FirstLBR << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "invalid incoming LBR: " <<
FirstLBR << '\n'; } } while (false)
;
893 }
894 }
895
896 // Fill out information for fall-through edges. The From and To could be
897 // within the same basic block, e.g. when two call instructions are in the
898 // same block. In this case we skip the processing.
899 if (FromBB == ToBB)
900 return true;
901
902 // Process blocks in the original layout order.
903 BinaryBasicBlock *BB = BF.getLayout().getBlock(FromBB->getIndex());
904 assert(BB == FromBB && "index mismatch")(static_cast <bool> (BB == FromBB && "index mismatch"
) ? void (0) : __assert_fail ("BB == FromBB && \"index mismatch\""
, "bolt/lib/Profile/DataAggregator.cpp", 904, __extension__ __PRETTY_FUNCTION__
))
;
905 while (BB != ToBB) {
906 BinaryBasicBlock *NextBB = BF.getLayout().getBlock(BB->getIndex() + 1);
907 assert((NextBB && NextBB->getOffset() > BB->getOffset()) && "bad layout")(static_cast <bool> ((NextBB && NextBB->getOffset
() > BB->getOffset()) && "bad layout") ? void (
0) : __assert_fail ("(NextBB && NextBB->getOffset() > BB->getOffset()) && \"bad layout\""
, "bolt/lib/Profile/DataAggregator.cpp", 907, __extension__ __PRETTY_FUNCTION__
))
;
908
909 // Check for bad LBRs.
910 if (!BB->getSuccessor(NextBB->getLabel())) {
911 LLVM_DEBUG(dbgs() << "no fall-through for the trace:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "no fall-through for the trace:\n"
<< " " << FirstLBR << '\n' << " " <<
SecondLBR << '\n'; } } while (false)
912 << " " << FirstLBR << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "no fall-through for the trace:\n"
<< " " << FirstLBR << '\n' << " " <<
SecondLBR << '\n'; } } while (false)
913 << " " << SecondLBR << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "no fall-through for the trace:\n"
<< " " << FirstLBR << '\n' << " " <<
SecondLBR << '\n'; } } while (false)
;
914 return false;
915 }
916
917 // Record fall-through jumps
918 BinaryBasicBlock::BinaryBranchInfo &BI = BB->getBranchInfo(*NextBB);
919 BI.Count += Count;
920
921 if (Branches) {
922 const MCInst *Instr = BB->getLastNonPseudoInstr();
923 uint64_t Offset = 0;
924 if (Instr)
925 Offset = BC.MIB->getOffsetWithDefault(*Instr, 0);
926 else
927 Offset = BB->getOffset();
928
929 Branches->emplace_back(Offset, NextBB->getOffset());
930 }
931
932 BB = NextBB;
933 }
934
935 return true;
936}
937
938Optional<SmallVector<std::pair<uint64_t, uint64_t>, 16>>
939DataAggregator::getFallthroughsInTrace(BinaryFunction &BF,
940 const LBREntry &FirstLBR,
941 const LBREntry &SecondLBR,
942 uint64_t Count) const {
943 SmallVector<std::pair<uint64_t, uint64_t>, 16> Res;
944
945 if (!recordTrace(BF, FirstLBR, SecondLBR, Count, &Res))
946 return std::nullopt;
947
948 return Res;
949}
950
951bool DataAggregator::recordEntry(BinaryFunction &BF, uint64_t To, bool Mispred,
952 uint64_t Count) const {
953 if (To > BF.getSize())
954 return false;
955
956 if (!BF.hasProfile())
957 BF.ExecutionCount = 0;
958
959 BinaryBasicBlock *EntryBB = nullptr;
960 if (To == 0) {
961 BF.ExecutionCount += Count;
962 if (!BF.empty())
963 EntryBB = &BF.front();
964 } else if (BinaryBasicBlock *BB = BF.getBasicBlockAtOffset(To)) {
965 if (BB->isEntryPoint())
966 EntryBB = BB;
967 }
968
969 if (EntryBB)
970 EntryBB->setExecutionCount(EntryBB->getKnownExecutionCount() + Count);
971
972 return true;
973}
974
975bool DataAggregator::recordExit(BinaryFunction &BF, uint64_t From, bool Mispred,
976 uint64_t Count) const {
977 if (!BF.isSimple() || From > BF.getSize())
978 return false;
979
980 if (!BF.hasProfile())
981 BF.ExecutionCount = 0;
982
983 return true;
984}
985
986ErrorOr<LBREntry> DataAggregator::parseLBREntry() {
987 LBREntry Res;
988 ErrorOr<StringRef> FromStrRes = parseString('/');
989 if (std::error_code EC = FromStrRes.getError())
990 return EC;
991 StringRef OffsetStr = FromStrRes.get();
992 if (OffsetStr.getAsInteger(0, Res.From)) {
993 reportError("expected hexadecimal number with From address");
994 Diag << "Found: " << OffsetStr << "\n";
995 return make_error_code(llvm::errc::io_error);
996 }
997
998 ErrorOr<StringRef> ToStrRes = parseString('/');
999 if (std::error_code EC = ToStrRes.getError())
1000 return EC;
1001 OffsetStr = ToStrRes.get();
1002 if (OffsetStr.getAsInteger(0, Res.To)) {
1003 reportError("expected hexadecimal number with To address");
1004 Diag << "Found: " << OffsetStr << "\n";
1005 return make_error_code(llvm::errc::io_error);
1006 }
1007
1008 ErrorOr<StringRef> MispredStrRes = parseString('/');
1009 if (std::error_code EC = MispredStrRes.getError())
1010 return EC;
1011 StringRef MispredStr = MispredStrRes.get();
1012 if (MispredStr.size() != 1 ||
1013 (MispredStr[0] != 'P' && MispredStr[0] != 'M' && MispredStr[0] != '-')) {
1014 reportError("expected single char for mispred bit");
1015 Diag << "Found: " << MispredStr << "\n";
1016 return make_error_code(llvm::errc::io_error);
1017 }
1018 Res.Mispred = MispredStr[0] == 'M';
1019
1020 static bool MispredWarning = true;
1021 if (MispredStr[0] == '-' && MispredWarning) {
1022 errs() << "PERF2BOLT-WARNING: misprediction bit is missing in profile\n";
1023 MispredWarning = false;
1024 }
1025
1026 ErrorOr<StringRef> Rest = parseString(FieldSeparator, true);
1027 if (std::error_code EC = Rest.getError())
1028 return EC;
1029 if (Rest.get().size() < 5) {
1030 reportError("expected rest of LBR entry");
1031 Diag << "Found: " << Rest.get() << "\n";
1032 return make_error_code(llvm::errc::io_error);
1033 }
1034 return Res;
1035}
1036
1037bool DataAggregator::checkAndConsumeFS() {
1038 if (ParsingBuf[0] != FieldSeparator)
1039 return false;
1040
1041 ParsingBuf = ParsingBuf.drop_front(1);
1042 Col += 1;
1043 return true;
1044}
1045
1046void DataAggregator::consumeRestOfLine() {
1047 size_t LineEnd = ParsingBuf.find_first_of('\n');
1048 if (LineEnd == StringRef::npos) {
1049 ParsingBuf = StringRef();
1050 Col = 0;
1051 Line += 1;
1052 return;
1053 }
1054 ParsingBuf = ParsingBuf.drop_front(LineEnd + 1);
1055 Col = 0;
1056 Line += 1;
1057}
1058
1059bool DataAggregator::checkNewLine() {
1060 return ParsingBuf[0] == '\n';
1061}
1062
1063ErrorOr<DataAggregator::PerfBranchSample> DataAggregator::parseBranchSample() {
1064 PerfBranchSample Res;
1065
1066 while (checkAndConsumeFS()) {
1067 }
1068
1069 ErrorOr<int64_t> PIDRes = parseNumberField(FieldSeparator, true);
1070 if (std::error_code EC = PIDRes.getError())
1071 return EC;
1072 auto MMapInfoIter = BinaryMMapInfo.find(*PIDRes);
1073 if (!opts::LinuxKernelMode && MMapInfoIter == BinaryMMapInfo.end()) {
1074 consumeRestOfLine();
1075 return make_error_code(errc::no_such_process);
1076 }
1077
1078 while (checkAndConsumeFS()) {
1079 }
1080
1081 ErrorOr<uint64_t> PCRes = parseHexField(FieldSeparator, true);
1082 if (std::error_code EC = PCRes.getError())
1083 return EC;
1084 Res.PC = PCRes.get();
1085
1086 if (checkAndConsumeNewLine())
1087 return Res;
1088
1089 while (!checkAndConsumeNewLine()) {
1090 checkAndConsumeFS();
1091
1092 ErrorOr<LBREntry> LBRRes = parseLBREntry();
1093 if (std::error_code EC = LBRRes.getError())
1094 return EC;
1095 LBREntry LBR = LBRRes.get();
1096 if (ignoreKernelInterrupt(LBR))
1097 continue;
1098 if (!BC->HasFixedLoadAddress)
1099 adjustLBR(LBR, MMapInfoIter->second);
1100 Res.LBR.push_back(LBR);
1101 }
1102
1103 return Res;
1104}
1105
1106ErrorOr<DataAggregator::PerfBasicSample> DataAggregator::parseBasicSample() {
1107 while (checkAndConsumeFS()) {
1108 }
1109
1110 ErrorOr<int64_t> PIDRes = parseNumberField(FieldSeparator, true);
1111 if (std::error_code EC = PIDRes.getError())
1112 return EC;
1113
1114 auto MMapInfoIter = BinaryMMapInfo.find(*PIDRes);
1115 if (MMapInfoIter == BinaryMMapInfo.end()) {
1116 consumeRestOfLine();
1117 return PerfBasicSample{StringRef(), 0};
1118 }
1119
1120 while (checkAndConsumeFS()) {
1121 }
1122
1123 ErrorOr<StringRef> Event = parseString(FieldSeparator);
1124 if (std::error_code EC = Event.getError())
1125 return EC;
1126
1127 while (checkAndConsumeFS()) {
1128 }
1129
1130 ErrorOr<uint64_t> AddrRes = parseHexField(FieldSeparator, true);
1131 if (std::error_code EC = AddrRes.getError())
1132 return EC;
1133
1134 if (!checkAndConsumeNewLine()) {
1135 reportError("expected end of line");
1136 return make_error_code(llvm::errc::io_error);
1137 }
1138
1139 uint64_t Address = *AddrRes;
1140 if (!BC->HasFixedLoadAddress)
1141 adjustAddress(Address, MMapInfoIter->second);
1142
1143 return PerfBasicSample{Event.get(), Address};
1144}
1145
1146ErrorOr<DataAggregator::PerfMemSample> DataAggregator::parseMemSample() {
1147 PerfMemSample Res{0, 0};
1148
1149 while (checkAndConsumeFS()) {
1150 }
1151
1152 ErrorOr<int64_t> PIDRes = parseNumberField(FieldSeparator, true);
1153 if (std::error_code EC = PIDRes.getError())
1154 return EC;
1155
1156 auto MMapInfoIter = BinaryMMapInfo.find(*PIDRes);
1157 if (MMapInfoIter == BinaryMMapInfo.end()) {
1158 consumeRestOfLine();
1159 return Res;
1160 }
1161
1162 while (checkAndConsumeFS()) {
1163 }
1164
1165 ErrorOr<StringRef> Event = parseString(FieldSeparator);
1166 if (std::error_code EC = Event.getError())
1167 return EC;
1168 if (!Event.get().contains("mem-loads")) {
1169 consumeRestOfLine();
1170 return Res;
1171 }
1172
1173 while (checkAndConsumeFS()) {
1174 }
1175
1176 ErrorOr<uint64_t> AddrRes = parseHexField(FieldSeparator);
1177 if (std::error_code EC = AddrRes.getError())
1178 return EC;
1179
1180 while (checkAndConsumeFS()) {
1181 }
1182
1183 ErrorOr<uint64_t> PCRes = parseHexField(FieldSeparator, true);
1184 if (std::error_code EC = PCRes.getError()) {
1185 consumeRestOfLine();
1186 return EC;
1187 }
1188
1189 if (!checkAndConsumeNewLine()) {
1190 reportError("expected end of line");
1191 return make_error_code(llvm::errc::io_error);
1192 }
1193
1194 uint64_t Address = *AddrRes;
1195 if (!BC->HasFixedLoadAddress)
1196 adjustAddress(Address, MMapInfoIter->second);
1197
1198 return PerfMemSample{PCRes.get(), Address};
1199}
1200
1201ErrorOr<Location> DataAggregator::parseLocationOrOffset() {
1202 auto parseOffset = [this]() -> ErrorOr<Location> {
1203 ErrorOr<uint64_t> Res = parseHexField(FieldSeparator);
1204 if (std::error_code EC = Res.getError())
1205 return EC;
1206 return Location(Res.get());
1207 };
1208
1209 size_t Sep = ParsingBuf.find_first_of(" \n");
1210 if (Sep == StringRef::npos)
1211 return parseOffset();
1212 StringRef LookAhead = ParsingBuf.substr(0, Sep);
1213 if (LookAhead.find_first_of(":") == StringRef::npos)
1214 return parseOffset();
1215
1216 ErrorOr<StringRef> BuildID = parseString(':');
1217 if (std::error_code EC = BuildID.getError())
1218 return EC;
1219 ErrorOr<uint64_t> Offset = parseHexField(FieldSeparator);
1220 if (std::error_code EC = Offset.getError())
1221 return EC;
1222 return Location(true, BuildID.get(), Offset.get());
1223}
1224
1225ErrorOr<DataAggregator::AggregatedLBREntry>
1226DataAggregator::parseAggregatedLBREntry() {
1227 while (checkAndConsumeFS()) {
1228 }
1229
1230 ErrorOr<StringRef> TypeOrErr = parseString(FieldSeparator);
1231 if (std::error_code EC = TypeOrErr.getError())
1232 return EC;
1233 auto Type = AggregatedLBREntry::BRANCH;
1234 if (TypeOrErr.get() == "B") {
1235 Type = AggregatedLBREntry::BRANCH;
1236 } else if (TypeOrErr.get() == "F") {
1237 Type = AggregatedLBREntry::FT;
1238 } else if (TypeOrErr.get() == "f") {
1239 Type = AggregatedLBREntry::FT_EXTERNAL_ORIGIN;
1240 } else {
1241 reportError("expected B, F or f");
1242 return make_error_code(llvm::errc::io_error);
1243 }
1244
1245 while (checkAndConsumeFS()) {
1246 }
1247 ErrorOr<Location> From = parseLocationOrOffset();
1248 if (std::error_code EC = From.getError())
1249 return EC;
1250
1251 while (checkAndConsumeFS()) {
1252 }
1253 ErrorOr<Location> To = parseLocationOrOffset();
1254 if (std::error_code EC = To.getError())
1255 return EC;
1256
1257 while (checkAndConsumeFS()) {
1258 }
1259 ErrorOr<int64_t> Frequency =
1260 parseNumberField(FieldSeparator, Type != AggregatedLBREntry::BRANCH);
1261 if (std::error_code EC = Frequency.getError())
1262 return EC;
1263
1264 uint64_t Mispreds = 0;
1265 if (Type == AggregatedLBREntry::BRANCH) {
1266 while (checkAndConsumeFS()) {
1267 }
1268 ErrorOr<int64_t> MispredsOrErr = parseNumberField(FieldSeparator, true);
1269 if (std::error_code EC = MispredsOrErr.getError())
1270 return EC;
1271 Mispreds = static_cast<uint64_t>(MispredsOrErr.get());
1272 }
1273
1274 if (!checkAndConsumeNewLine()) {
1275 reportError("expected end of line");
1276 return make_error_code(llvm::errc::io_error);
1277 }
1278
1279 return AggregatedLBREntry{From.get(), To.get(),
1280 static_cast<uint64_t>(Frequency.get()), Mispreds,
1281 Type};
1282}
1283
1284bool DataAggregator::ignoreKernelInterrupt(LBREntry &LBR) const {
1285 return opts::IgnoreInterruptLBR &&
1286 (LBR.From >= KernelBaseAddr || LBR.To >= KernelBaseAddr);
1287}
1288
1289std::error_code DataAggregator::printLBRHeatMap() {
1290 outs() << "PERF2BOLT: parse branch events...\n";
1291 NamedRegionTimer T("parseBranch", "Parsing branch events", TimerGroupName,
1292 TimerGroupDesc, opts::TimeAggregator);
1293
1294 if (opts::LinuxKernelMode) {
1295 opts::HeatmapMaxAddress = 0xffffffffffffffff;
1296 opts::HeatmapMinAddress = KernelBaseAddr;
1297 }
1298 Heatmap HM(opts::HeatmapBlock, opts::HeatmapMinAddress,
1299 opts::HeatmapMaxAddress, getTextSections(BC));
1300 uint64_t NumTotalSamples = 0;
1301
1302 if (opts::BasicAggregation) {
1303 while (hasData()) {
1304 ErrorOr<PerfBasicSample> SampleRes = parseBasicSample();
1305 if (std::error_code EC = SampleRes.getError()) {
1306 if (EC == errc::no_such_process)
1307 continue;
1308 return EC;
1309 }
1310 PerfBasicSample &Sample = SampleRes.get();
1311 HM.registerAddress(Sample.PC);
1312 NumTotalSamples++;
1313 }
1314 outs() << "HEATMAP: read " << NumTotalSamples << " basic samples\n";
1315 } else {
1316 while (hasData()) {
1317 ErrorOr<PerfBranchSample> SampleRes = parseBranchSample();
1318 if (std::error_code EC = SampleRes.getError()) {
1319 if (EC == errc::no_such_process)
1320 continue;
1321 return EC;
1322 }
1323
1324 PerfBranchSample &Sample = SampleRes.get();
1325
1326 // LBRs are stored in reverse execution order. NextLBR refers to the next
1327 // executed branch record.
1328 const LBREntry *NextLBR = nullptr;
1329 for (const LBREntry &LBR : Sample.LBR) {
1330 if (NextLBR) {
1331 // Record fall-through trace.
1332 const uint64_t TraceFrom = LBR.To;
1333 const uint64_t TraceTo = NextLBR->From;
1334 ++FallthroughLBRs[Trace(TraceFrom, TraceTo)].InternCount;
1335 }
1336 NextLBR = &LBR;
1337 }
1338 if (!Sample.LBR.empty()) {
1339 HM.registerAddress(Sample.LBR.front().To);
1340 HM.registerAddress(Sample.LBR.back().From);
1341 }
1342 NumTotalSamples += Sample.LBR.size();
1343 }
1344 outs() << "HEATMAP: read " << NumTotalSamples << " LBR samples\n";
1345 outs() << "HEATMAP: " << FallthroughLBRs.size() << " unique traces\n";
1346 }
1347
1348 if (!NumTotalSamples) {
1349 if (opts::BasicAggregation) {
1350 errs() << "HEATMAP-ERROR: no basic event samples detected in profile. "
1351 "Cannot build heatmap.";
1352 } else {
1353 errs() << "HEATMAP-ERROR: no LBR traces detected in profile. "
1354 "Cannot build heatmap. Use -nl for building heatmap from "
1355 "basic events.\n";
1356 }
1357 exit(1);
1358 }
1359
1360 outs() << "HEATMAP: building heat map...\n";
1361
1362 for (const auto &LBR : FallthroughLBRs) {
1363 const Trace &Trace = LBR.first;
1364 const FTInfo &Info = LBR.second;
1365 HM.registerAddressRange(Trace.From, Trace.To, Info.InternCount);
1366 }
1367
1368 if (HM.getNumInvalidRanges())
1369 outs() << "HEATMAP: invalid traces: " << HM.getNumInvalidRanges() << '\n';
1370
1371 if (!HM.size()) {
1372 errs() << "HEATMAP-ERROR: no valid traces registered\n";
1373 exit(1);
1374 }
1375
1376 HM.print(opts::OutputFilename);
1377 if (opts::OutputFilename == "-")
1378 HM.printCDF(opts::OutputFilename);
1379 else
1380 HM.printCDF(opts::OutputFilename + ".csv");
1381 if (opts::OutputFilename == "-")
1382 HM.printSectionHotness(opts::OutputFilename);
1383 else
1384 HM.printSectionHotness(opts::OutputFilename + "-section-hotness.csv");
1385
1386 return std::error_code();
1387}
1388
1389std::error_code DataAggregator::parseBranchEvents() {
1390 outs() << "PERF2BOLT: parse branch events...\n";
1391 NamedRegionTimer T("parseBranch", "Parsing branch events", TimerGroupName,
1392 TimerGroupDesc, opts::TimeAggregator);
1393
1394 uint64_t NumTotalSamples = 0;
1395 uint64_t NumEntries = 0;
1396 uint64_t NumSamples = 0;
1397 uint64_t NumSamplesNoLBR = 0;
1398 uint64_t NumTraces = 0;
1399 bool NeedsSkylakeFix = false;
1400
1401 while (hasData() && NumTotalSamples < opts::MaxSamples) {
1402 ++NumTotalSamples;
1403
1404 ErrorOr<PerfBranchSample> SampleRes = parseBranchSample();
1405 if (std::error_code EC = SampleRes.getError()) {
1406 if (EC == errc::no_such_process)
1407 continue;
1408 return EC;
1409 }
1410 ++NumSamples;
1411
1412 PerfBranchSample &Sample = SampleRes.get();
1413 if (opts::WriteAutoFDOData)
1414 ++BasicSamples[Sample.PC];
1415
1416 if (Sample.LBR.empty()) {
1417 ++NumSamplesNoLBR;
1418 continue;
1419 }
1420
1421 NumEntries += Sample.LBR.size();
1422 if (BAT && Sample.LBR.size() == 32 && !NeedsSkylakeFix) {
1423 errs() << "PERF2BOLT-WARNING: using Intel Skylake bug workaround\n";
1424 NeedsSkylakeFix = true;
1425 }
1426
1427 // LBRs are stored in reverse execution order. NextPC refers to the next
1428 // recorded executed PC.
1429 uint64_t NextPC = opts::UseEventPC ? Sample.PC : 0;
1430 uint32_t NumEntry = 0;
1431 for (const LBREntry &LBR : Sample.LBR) {
1432 ++NumEntry;
1433 // Hardware bug workaround: Intel Skylake (which has 32 LBR entries)
1434 // sometimes record entry 32 as an exact copy of entry 31. This will cause
1435 // us to likely record an invalid trace and generate a stale function for
1436 // BAT mode (non BAT disassembles the function and is able to ignore this
1437 // trace at aggregation time). Drop first 2 entries (last two, in
1438 // chronological order)
1439 if (NeedsSkylakeFix && NumEntry <= 2)
1440 continue;
1441 if (NextPC) {
1442 // Record fall-through trace.
1443 const uint64_t TraceFrom = LBR.To;
1444 const uint64_t TraceTo = NextPC;
1445 const BinaryFunction *TraceBF =
1446 getBinaryFunctionContainingAddress(TraceFrom);
1447 if (TraceBF && TraceBF->containsAddress(TraceTo)) {
1448 FTInfo &Info = FallthroughLBRs[Trace(TraceFrom, TraceTo)];
1449 if (TraceBF->containsAddress(LBR.From))
1450 ++Info.InternCount;
1451 else
1452 ++Info.ExternCount;
1453 } else {
1454 if (TraceBF && getBinaryFunctionContainingAddress(TraceTo)) {
1455 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< TraceBF->getPrintName() << " @ " << Twine
::utohexstr(TraceFrom - TraceBF->getAddress()) << " and ending @ "
<< Twine::utohexstr(TraceTo) << '\n'; } } while (
false)
1456 << "Invalid trace starting in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< TraceBF->getPrintName() << " @ " << Twine
::utohexstr(TraceFrom - TraceBF->getAddress()) << " and ending @ "
<< Twine::utohexstr(TraceTo) << '\n'; } } while (
false)
1457 << TraceBF->getPrintName() << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< TraceBF->getPrintName() << " @ " << Twine
::utohexstr(TraceFrom - TraceBF->getAddress()) << " and ending @ "
<< Twine::utohexstr(TraceTo) << '\n'; } } while (
false)
1458 << Twine::utohexstr(TraceFrom - TraceBF->getAddress())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< TraceBF->getPrintName() << " @ " << Twine
::utohexstr(TraceFrom - TraceBF->getAddress()) << " and ending @ "
<< Twine::utohexstr(TraceTo) << '\n'; } } while (
false)
1459 << " and ending @ " << Twine::utohexstr(TraceTo)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< TraceBF->getPrintName() << " @ " << Twine
::utohexstr(TraceFrom - TraceBF->getAddress()) << " and ending @ "
<< Twine::utohexstr(TraceTo) << '\n'; } } while (
false)
1460 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Invalid trace starting in "
<< TraceBF->getPrintName() << " @ " << Twine
::utohexstr(TraceFrom - TraceBF->getAddress()) << " and ending @ "
<< Twine::utohexstr(TraceTo) << '\n'; } } while (
false)
;
1461 ++NumInvalidTraces;
1462 } else {
1463 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1464 << "Out of range trace starting in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1465 << (TraceBF ? TraceBF->getPrintName() : "None") << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1466 << Twine::utohexstr(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1467 TraceFrom - (TraceBF ? TraceBF->getAddress() : 0))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1468 << " and ending in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1469 << (getBinaryFunctionContainingAddress(TraceTo)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1470 ? getBinaryFunctionContainingAddress(TraceTo)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1471 ->getPrintName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1472 : "None")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1473 << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1474 << Twine::utohexstr(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1475 TraceTo -do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1476 (getBinaryFunctionContainingAddress(TraceTo)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1477 ? getBinaryFunctionContainingAddress(TraceTo)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1478 ->getAddress()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1479 : 0))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
1480 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Out of range trace starting in "
<< (TraceBF ? TraceBF->getPrintName() : "None") <<
" @ " << Twine::utohexstr( TraceFrom - (TraceBF ? TraceBF
->getAddress() : 0)) << " and ending in " << (
getBinaryFunctionContainingAddress(TraceTo) ? getBinaryFunctionContainingAddress
(TraceTo) ->getPrintName() : "None") << " @ " <<
Twine::utohexstr( TraceTo - (getBinaryFunctionContainingAddress
(TraceTo) ? getBinaryFunctionContainingAddress(TraceTo) ->
getAddress() : 0)) << '\n'; } } while (false)
;
1481 ++NumLongRangeTraces;
1482 }
1483 }
1484 ++NumTraces;
1485 }
1486 NextPC = LBR.From;
1487
1488 uint64_t From = LBR.From;
1489 if (!getBinaryFunctionContainingAddress(From))
1490 From = 0;
1491 uint64_t To = LBR.To;
1492 if (!getBinaryFunctionContainingAddress(To))
1493 To = 0;
1494 if (!From && !To)
1495 continue;
1496 BranchInfo &Info = BranchLBRs[Trace(From, To)];
1497 ++Info.TakenCount;
1498 Info.MispredCount += LBR.Mispred;
1499 }
1500 }
1501
1502 for (const auto &LBR : BranchLBRs) {
1503 const Trace &Trace = LBR.first;
1504 if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Trace.From))
1505 BF->setHasProfileAvailable();
1506 if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Trace.To))
1507 BF->setHasProfileAvailable();
1508 }
1509
1510 auto printColored = [](raw_ostream &OS, float Percent, float T1, float T2) {
1511 OS << " (";
1512 if (OS.has_colors()) {
1513 if (Percent > T2)
1514 OS.changeColor(raw_ostream::RED);
1515 else if (Percent > T1)
1516 OS.changeColor(raw_ostream::YELLOW);
1517 else
1518 OS.changeColor(raw_ostream::GREEN);
1519 }
1520 OS << format("%.1f%%", Percent);
1521 if (OS.has_colors())
1522 OS.resetColor();
1523 OS << ")";
1524 };
1525
1526 outs() << "PERF2BOLT: read " << NumSamples << " samples and " << NumEntries
1527 << " LBR entries\n";
1528 if (NumTotalSamples) {
1529 if (NumSamples && NumSamplesNoLBR == NumSamples) {
1530 // Note: we don't know if perf2bolt is being used to parse memory samples
1531 // at this point. In this case, it is OK to parse zero LBRs.
1532 errs() << "PERF2BOLT-WARNING: all recorded samples for this binary lack "
1533 "LBR. Record profile with perf record -j any or run perf2bolt "
1534 "in no-LBR mode with -nl (the performance improvement in -nl "
1535 "mode may be limited)\n";
1536 } else {
1537 const uint64_t IgnoredSamples = NumTotalSamples - NumSamples;
1538 const float PercentIgnored = 100.0f * IgnoredSamples / NumTotalSamples;
1539 outs() << "PERF2BOLT: " << IgnoredSamples << " samples";
1540 printColored(outs(), PercentIgnored, 20, 50);
1541 outs() << " were ignored\n";
1542 if (PercentIgnored > 50.0f)
1543 errs() << "PERF2BOLT-WARNING: less than 50% of all recorded samples "
1544 "were attributed to the input binary\n";
1545 }
1546 }
1547 outs() << "PERF2BOLT: traces mismatching disassembled function contents: "
1548 << NumInvalidTraces;
1549 float Perc = 0.0f;
1550 if (NumTraces > 0) {
1551 Perc = NumInvalidTraces * 100.0f / NumTraces;
1552 printColored(outs(), Perc, 5, 10);
1553 }
1554 outs() << "\n";
1555 if (Perc > 10.0f)
1556 outs() << "\n !! WARNING !! This high mismatch ratio indicates the input "
1557 "binary is probably not the same binary used during profiling "
1558 "collection. The generated data may be ineffective for improving "
1559 "performance.\n\n";
1560
1561 outs() << "PERF2BOLT: out of range traces involving unknown regions: "
1562 << NumLongRangeTraces;
1563 if (NumTraces > 0)
1564 outs() << format(" (%.1f%%)", NumLongRangeTraces * 100.0f / NumTraces);
1565 outs() << "\n";
1566
1567 if (NumColdSamples > 0) {
1568 const float ColdSamples = NumColdSamples * 100.0f / NumTotalSamples;
1569 outs() << "PERF2BOLT: " << NumColdSamples
1570 << format(" (%.1f%%)", ColdSamples)
1571 << " samples recorded in cold regions of split functions.\n";
1572 if (ColdSamples > 5.0f)
1573 outs()
1574 << "WARNING: The BOLT-processed binary where samples were collected "
1575 "likely used bad data or your service observed a large shift in "
1576 "profile. You may want to audit this.\n";
1577 }
1578
1579 return std::error_code();
1580}
1581
1582void DataAggregator::processBranchEvents() {
1583 outs() << "PERF2BOLT: processing branch events...\n";
1584 NamedRegionTimer T("processBranch", "Processing branch events",
1585 TimerGroupName, TimerGroupDesc, opts::TimeAggregator);
1586
1587 for (const auto &AggrLBR : FallthroughLBRs) {
1588 const Trace &Loc = AggrLBR.first;
1589 const FTInfo &Info = AggrLBR.second;
1590 LBREntry First{Loc.From, Loc.From, false};
1591 LBREntry Second{Loc.To, Loc.To, false};
1592 if (Info.InternCount)
1593 doTrace(First, Second, Info.InternCount);
1594 if (Info.ExternCount) {
1595 First.From = 0;
1596 doTrace(First, Second, Info.ExternCount);
1597 }
1598 }
1599
1600 for (const auto &AggrLBR : BranchLBRs) {
1601 const Trace &Loc = AggrLBR.first;
1602 const BranchInfo &Info = AggrLBR.second;
1603 doBranch(Loc.From, Loc.To, Info.TakenCount, Info.MispredCount);
1604 }
1605}
1606
1607std::error_code DataAggregator::parseBasicEvents() {
1608 outs() << "PERF2BOLT: parsing basic events (without LBR)...\n";
1609 NamedRegionTimer T("parseBasic", "Parsing basic events", TimerGroupName,
1610 TimerGroupDesc, opts::TimeAggregator);
1611 while (hasData()) {
1612 ErrorOr<PerfBasicSample> Sample = parseBasicSample();
1613 if (std::error_code EC = Sample.getError())
1614 return EC;
1615
1616 if (!Sample->PC)
1617 continue;
1618
1619 if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC))
1620 BF->setHasProfileAvailable();
1621
1622 ++BasicSamples[Sample->PC];
1623 EventNames.insert(Sample->EventName);
1624 }
1625
1626 return std::error_code();
1627}
1628
1629void DataAggregator::processBasicEvents() {
1630 outs() << "PERF2BOLT: processing basic events (without LBR)...\n";
1631 NamedRegionTimer T("processBasic", "Processing basic events", TimerGroupName,
1632 TimerGroupDesc, opts::TimeAggregator);
1633 uint64_t OutOfRangeSamples = 0;
1634 uint64_t NumSamples = 0;
1635 for (auto &Sample : BasicSamples) {
1636 const uint64_t PC = Sample.first;
1637 const uint64_t HitCount = Sample.second;
1638 NumSamples += HitCount;
1639 BinaryFunction *Func = getBinaryFunctionContainingAddress(PC);
1640 if (!Func) {
1641 OutOfRangeSamples += HitCount;
1642 continue;
1643 }
1644
1645 doSample(*Func, PC, HitCount);
1646 }
1647 outs() << "PERF2BOLT: read " << NumSamples << " samples\n";
1648
1649 outs() << "PERF2BOLT: out of range samples recorded in unknown regions: "
1650 << OutOfRangeSamples;
1651 float Perc = 0.0f;
1652 if (NumSamples > 0) {
1653 outs() << " (";
1654 Perc = OutOfRangeSamples * 100.0f / NumSamples;
1655 if (outs().has_colors()) {
1656 if (Perc > 60.0f)
1657 outs().changeColor(raw_ostream::RED);
1658 else if (Perc > 40.0f)
1659 outs().changeColor(raw_ostream::YELLOW);
1660 else
1661 outs().changeColor(raw_ostream::GREEN);
1662 }
1663 outs() << format("%.1f%%", Perc);
1664 if (outs().has_colors())
1665 outs().resetColor();
1666 outs() << ")";
1667 }
1668 outs() << "\n";
1669 if (Perc > 80.0f)
1670 outs() << "\n !! WARNING !! This high mismatch ratio indicates the input "
1671 "binary is probably not the same binary used during profiling "
1672 "collection. The generated data may be ineffective for improving "
1673 "performance.\n\n";
1674}
1675
1676std::error_code DataAggregator::parseMemEvents() {
1677 outs() << "PERF2BOLT: parsing memory events...\n";
1678 NamedRegionTimer T("parseMemEvents", "Parsing mem events", TimerGroupName,
1679 TimerGroupDesc, opts::TimeAggregator);
1680 while (hasData()) {
1681 ErrorOr<PerfMemSample> Sample = parseMemSample();
1682 if (std::error_code EC = Sample.getError())
1683 return EC;
1684
1685 if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC))
1686 BF->setHasProfileAvailable();
1687
1688 MemSamples.emplace_back(std::move(Sample.get()));
1689 }
1690
1691 return std::error_code();
1692}
1693
1694void DataAggregator::processMemEvents() {
1695 NamedRegionTimer T("ProcessMemEvents", "Processing mem events",
1696 TimerGroupName, TimerGroupDesc, opts::TimeAggregator);
1697 for (const PerfMemSample &Sample : MemSamples) {
1698 uint64_t PC = Sample.PC;
1699 uint64_t Addr = Sample.Addr;
1700 StringRef FuncName;
1701 StringRef MemName;
1702
1703 // Try to resolve symbol for PC
1704 BinaryFunction *Func = getBinaryFunctionContainingAddress(PC);
1705 if (!Func) {
1706 LLVM_DEBUG(if (PC != 0) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { if (PC != 0) { dbgs() << "Skipped mem event: 0x"
<< Twine::utohexstr(PC) << " => 0x" << Twine
::utohexstr(Addr) << "\n"; }; } } while (false)
1707 dbgs() << "Skipped mem event: 0x" << Twine::utohexstr(PC) << " => 0x"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { if (PC != 0) { dbgs() << "Skipped mem event: 0x"
<< Twine::utohexstr(PC) << " => 0x" << Twine
::utohexstr(Addr) << "\n"; }; } } while (false)
1708 << Twine::utohexstr(Addr) << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { if (PC != 0) { dbgs() << "Skipped mem event: 0x"
<< Twine::utohexstr(PC) << " => 0x" << Twine
::utohexstr(Addr) << "\n"; }; } } while (false)
1709 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { if (PC != 0) { dbgs() << "Skipped mem event: 0x"
<< Twine::utohexstr(PC) << " => 0x" << Twine
::utohexstr(Addr) << "\n"; }; } } while (false)
;
1710 continue;
1711 }
1712
1713 FuncName = Func->getOneName();
1714 PC -= Func->getAddress();
1715
1716 // Try to resolve symbol for memory load
1717 if (BinaryData *BD = BC->getBinaryDataContainingAddress(Addr)) {
1718 MemName = BD->getName();
1719 Addr -= BD->getAddress();
1720 } else if (opts::FilterMemProfile) {
1721 // Filter out heap/stack accesses
1722 continue;
1723 }
1724
1725 const Location FuncLoc(!FuncName.empty(), FuncName, PC);
1726 const Location AddrLoc(!MemName.empty(), MemName, Addr);
1727
1728 FuncMemData *MemData = &NamesToMemEvents[FuncName];
1729 setMemData(*Func, MemData);
1730 MemData->update(FuncLoc, AddrLoc);
1731 LLVM_DEBUG(dbgs() << "Mem event: " << FuncLoc << " = " << AddrLoc << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { dbgs() << "Mem event: " << FuncLoc
<< " = " << AddrLoc << "\n"; } } while (false
)
;
1732 }
1733}
1734
1735std::error_code DataAggregator::parsePreAggregatedLBRSamples() {
1736 outs() << "PERF2BOLT: parsing pre-aggregated profile...\n";
1737 NamedRegionTimer T("parseAggregated", "Parsing aggregated branch events",
1738 TimerGroupName, TimerGroupDesc, opts::TimeAggregator);
1739 while (hasData()) {
1740 ErrorOr<AggregatedLBREntry> AggrEntry = parseAggregatedLBREntry();
1741 if (std::error_code EC = AggrEntry.getError())
1742 return EC;
1743
1744 if (BinaryFunction *BF =
1745 getBinaryFunctionContainingAddress(AggrEntry->From.Offset))
1746 BF->setHasProfileAvailable();
1747 if (BinaryFunction *BF =
1748 getBinaryFunctionContainingAddress(AggrEntry->To.Offset))
1749 BF->setHasProfileAvailable();
1750
1751 AggregatedLBRs.emplace_back(std::move(AggrEntry.get()));
1752 }
1753
1754 return std::error_code();
1755}
1756
1757void DataAggregator::processPreAggregated() {
1758 outs() << "PERF2BOLT: processing pre-aggregated profile...\n";
1759 NamedRegionTimer T("processAggregated", "Processing aggregated branch events",
1760 TimerGroupName, TimerGroupDesc, opts::TimeAggregator);
1761
1762 uint64_t NumTraces = 0;
1763 for (const AggregatedLBREntry &AggrEntry : AggregatedLBRs) {
1764 switch (AggrEntry.EntryType) {
5
Control jumps to 'case FT:' at line 1769
1765 case AggregatedLBREntry::BRANCH:
1766 doBranch(AggrEntry.From.Offset, AggrEntry.To.Offset, AggrEntry.Count,
1767 AggrEntry.Mispreds);
1768 break;
1769 case AggregatedLBREntry::FT:
1770 case AggregatedLBREntry::FT_EXTERNAL_ORIGIN: {
1771 LBREntry First{AggrEntry.EntryType
5.1
Field 'EntryType' is equal to FT
== AggregatedLBREntry::FT
6
'?' condition is true
1772 ? AggrEntry.From.Offset
1773 : 0,
1774 AggrEntry.From.Offset, false};
1775 LBREntry Second{AggrEntry.To.Offset, AggrEntry.To.Offset, false};
1776 doTrace(First, Second, AggrEntry.Count);
7
Calling 'DataAggregator::doTrace'
1777 NumTraces += AggrEntry.Count;
1778 break;
1779 }
1780 }
1781 }
1782
1783 outs() << "PERF2BOLT: read " << AggregatedLBRs.size()
1784 << " aggregated LBR entries\n";
1785 outs() << "PERF2BOLT: traces mismatching disassembled function contents: "
1786 << NumInvalidTraces;
1787 float Perc = 0.0f;
1788 if (NumTraces > 0) {
1789 outs() << " (";
1790 Perc = NumInvalidTraces * 100.0f / NumTraces;
1791 if (outs().has_colors()) {
1792 if (Perc > 10.0f)
1793 outs().changeColor(raw_ostream::RED);
1794 else if (Perc > 5.0f)
1795 outs().changeColor(raw_ostream::YELLOW);
1796 else
1797 outs().changeColor(raw_ostream::GREEN);
1798 }
1799 outs() << format("%.1f%%", Perc);
1800 if (outs().has_colors())
1801 outs().resetColor();
1802 outs() << ")";
1803 }
1804 outs() << "\n";
1805 if (Perc > 10.0f)
1806 outs() << "\n !! WARNING !! This high mismatch ratio indicates the input "
1807 "binary is probably not the same binary used during profiling "
1808 "collection. The generated data may be ineffective for improving "
1809 "performance.\n\n";
1810
1811 outs() << "PERF2BOLT: Out of range traces involving unknown regions: "
1812 << NumLongRangeTraces;
1813 if (NumTraces > 0)
1814 outs() << format(" (%.1f%%)", NumLongRangeTraces * 100.0f / NumTraces);
1815 outs() << "\n";
1816}
1817
1818Optional<int32_t> DataAggregator::parseCommExecEvent() {
1819 size_t LineEnd = ParsingBuf.find_first_of("\n");
1820 if (LineEnd == StringRef::npos) {
1821 reportError("expected rest of line");
1822 Diag << "Found: " << ParsingBuf << "\n";
1823 return std::nullopt;
1824 }
1825 StringRef Line = ParsingBuf.substr(0, LineEnd);
1826
1827 size_t Pos = Line.find("PERF_RECORD_COMM exec");
1828 if (Pos == StringRef::npos)
1829 return std::nullopt;
1830 Line = Line.drop_front(Pos);
1831
1832 // Line:
1833 // PERF_RECORD_COMM exec: <name>:<pid>/<tid>"
1834 StringRef PIDStr = Line.rsplit(':').second.split('/').first;
1835 int32_t PID;
1836 if (PIDStr.getAsInteger(10, PID)) {
1837 reportError("expected PID");
1838 Diag << "Found: " << PIDStr << "in '" << Line << "'\n";
1839 return std::nullopt;
1840 }
1841
1842 return PID;
1843}
1844
1845namespace {
1846Optional<uint64_t> parsePerfTime(const StringRef TimeStr) {
1847 const StringRef SecTimeStr = TimeStr.split('.').first;
1848 const StringRef USecTimeStr = TimeStr.split('.').second;
1849 uint64_t SecTime;
1850 uint64_t USecTime;
1851 if (SecTimeStr.getAsInteger(10, SecTime) ||
1852 USecTimeStr.getAsInteger(10, USecTime))
1853 return std::nullopt;
1854 return SecTime * 1000000ULL + USecTime;
1855}
1856}
1857
1858Optional<DataAggregator::ForkInfo> DataAggregator::parseForkEvent() {
1859 while (checkAndConsumeFS()) {
1860 }
1861
1862 size_t LineEnd = ParsingBuf.find_first_of("\n");
1863 if (LineEnd == StringRef::npos) {
1864 reportError("expected rest of line");
1865 Diag << "Found: " << ParsingBuf << "\n";
1866 return std::nullopt;
1867 }
1868 StringRef Line = ParsingBuf.substr(0, LineEnd);
1869
1870 size_t Pos = Line.find("PERF_RECORD_FORK");
1871 if (Pos == StringRef::npos) {
1872 consumeRestOfLine();
1873 return std::nullopt;
1874 }
1875
1876 ForkInfo FI;
1877
1878 const StringRef TimeStr =
1879 Line.substr(0, Pos).rsplit(':').first.rsplit(FieldSeparator).second;
1880 if (Optional<uint64_t> TimeRes = parsePerfTime(TimeStr)) {
1881 FI.Time = *TimeRes;
1882 }
1883
1884 Line = Line.drop_front(Pos);
1885
1886 // Line:
1887 // PERF_RECORD_FORK(<child_pid>:<child_tid>):(<parent_pid>:<parent_tid>)
1888 const StringRef ChildPIDStr = Line.split('(').second.split(':').first;
1889 if (ChildPIDStr.getAsInteger(10, FI.ChildPID)) {
1890 reportError("expected PID");
1891 Diag << "Found: " << ChildPIDStr << "in '" << Line << "'\n";
1892 return std::nullopt;
1893 }
1894
1895 const StringRef ParentPIDStr = Line.rsplit('(').second.split(':').first;
1896 if (ParentPIDStr.getAsInteger(10, FI.ParentPID)) {
1897 reportError("expected PID");
1898 Diag << "Found: " << ParentPIDStr << "in '" << Line << "'\n";
1899 return std::nullopt;
1900 }
1901
1902 consumeRestOfLine();
1903
1904 return FI;
1905}
1906
1907ErrorOr<std::pair<StringRef, DataAggregator::MMapInfo>>
1908DataAggregator::parseMMapEvent() {
1909 while (checkAndConsumeFS()) {
1910 }
1911
1912 MMapInfo ParsedInfo;
1913
1914 size_t LineEnd = ParsingBuf.find_first_of("\n");
1915 if (LineEnd == StringRef::npos) {
1916 reportError("expected rest of line");
1917 Diag << "Found: " << ParsingBuf << "\n";
1918 return make_error_code(llvm::errc::io_error);
1919 }
1920 StringRef Line = ParsingBuf.substr(0, LineEnd);
1921
1922 size_t Pos = Line.find("PERF_RECORD_MMAP2");
1923 if (Pos == StringRef::npos) {
1924 consumeRestOfLine();
1925 return std::make_pair(StringRef(), ParsedInfo);
1926 }
1927
1928 // Line:
1929 // {<name> .* <sec>.<usec>: }PERF_RECORD_MMAP2 <pid>/<tid>: .* <file_name>
1930
1931 const StringRef TimeStr =
1932 Line.substr(0, Pos).rsplit(':').first.rsplit(FieldSeparator).second;
1933 if (Optional<uint64_t> TimeRes = parsePerfTime(TimeStr))
1934 ParsedInfo.Time = *TimeRes;
1935
1936 Line = Line.drop_front(Pos);
1937
1938 // Line:
1939 // PERF_RECORD_MMAP2 <pid>/<tid>: [<hexbase>(<hexsize>) .*]: .* <file_name>
1940
1941 StringRef FileName = Line.rsplit(FieldSeparator).second;
1942 if (FileName.startswith("//") || FileName.startswith("[")) {
1943 consumeRestOfLine();
1944 return std::make_pair(StringRef(), ParsedInfo);
1945 }
1946 FileName = sys::path::filename(FileName);
1947
1948 const StringRef PIDStr = Line.split(FieldSeparator).second.split('/').first;
1949 if (PIDStr.getAsInteger(10, ParsedInfo.PID)) {
1950 reportError("expected PID");
1951 Diag << "Found: " << PIDStr << "in '" << Line << "'\n";
1952 return make_error_code(llvm::errc::io_error);
1953 }
1954
1955 const StringRef BaseAddressStr = Line.split('[').second.split('(').first;
1956 if (BaseAddressStr.getAsInteger(0, ParsedInfo.MMapAddress)) {
1957 reportError("expected base address");
1958 Diag << "Found: " << BaseAddressStr << "in '" << Line << "'\n";
1959 return make_error_code(llvm::errc::io_error);
1960 }
1961
1962 const StringRef SizeStr = Line.split('(').second.split(')').first;
1963 if (SizeStr.getAsInteger(0, ParsedInfo.Size)) {
1964 reportError("expected mmaped size");
1965 Diag << "Found: " << SizeStr << "in '" << Line << "'\n";
1966 return make_error_code(llvm::errc::io_error);
1967 }
1968
1969 const StringRef OffsetStr =
1970 Line.split('@').second.ltrim().split(FieldSeparator).first;
1971 if (OffsetStr.getAsInteger(0, ParsedInfo.Offset)) {
1972 reportError("expected mmaped page-aligned offset");
1973 Diag << "Found: " << OffsetStr << "in '" << Line << "'\n";
1974 return make_error_code(llvm::errc::io_error);
1975 }
1976
1977 consumeRestOfLine();
1978
1979 return std::make_pair(FileName, ParsedInfo);
1980}
1981
1982std::error_code DataAggregator::parseMMapEvents() {
1983 outs() << "PERF2BOLT: parsing perf-script mmap events output\n";
1984 NamedRegionTimer T("parseMMapEvents", "Parsing mmap events", TimerGroupName,
1985 TimerGroupDesc, opts::TimeAggregator);
1986
1987 std::multimap<StringRef, MMapInfo> GlobalMMapInfo;
1988 while (hasData()) {
1989 ErrorOr<std::pair<StringRef, MMapInfo>> FileMMapInfoRes = parseMMapEvent();
1990 if (std::error_code EC = FileMMapInfoRes.getError())
1991 return EC;
1992
1993 std::pair<StringRef, MMapInfo> FileMMapInfo = FileMMapInfoRes.get();
1994 if (FileMMapInfo.second.PID == -1)
1995 continue;
1996
1997 // Consider only the first mapping of the file for any given PID
1998 bool PIDExists = false;
1999 auto Range = GlobalMMapInfo.equal_range(FileMMapInfo.first);
2000 for (auto MI = Range.first; MI != Range.second; ++MI) {
2001 if (MI->second.PID == FileMMapInfo.second.PID) {
2002 PIDExists = true;
2003 break;
2004 }
2005 }
2006 if (PIDExists)
2007 continue;
2008
2009 GlobalMMapInfo.insert(FileMMapInfo);
2010 }
2011
2012 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
2013 dbgs() << "FileName -> mmap info:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
2014 for (const std::pair<const StringRef, MMapInfo> &Pair : GlobalMMapInfo)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
2015 dbgs() << " " << Pair.first << " : " << Pair.second.PID << " [0x"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
2016 << Twine::utohexstr(Pair.second.MMapAddress) << ", "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
2017 << Twine::utohexstr(Pair.second.Size) << " @ "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
2018 << Twine::utohexstr(Pair.second.Offset) << "]\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
2019 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { dbgs() << "FileName -> mmap info:\n"
; for (const std::pair<const StringRef, MMapInfo> &
Pair : GlobalMMapInfo) dbgs() << " " << Pair.first
<< " : " << Pair.second.PID << " [0x" <<
Twine::utohexstr(Pair.second.MMapAddress) << ", " <<
Twine::utohexstr(Pair.second.Size) << " @ " << Twine
::utohexstr(Pair.second.Offset) << "]\n"; }; } } while (
false)
;
2020
2021 StringRef NameToUse = llvm::sys::path::filename(BC->getFilename());
2022 if (GlobalMMapInfo.count(NameToUse) == 0 && !BuildIDBinaryName.empty()) {
2023 errs() << "PERF2BOLT-WARNING: using \"" << BuildIDBinaryName
2024 << "\" for profile matching\n";
2025 NameToUse = BuildIDBinaryName;
2026 }
2027
2028 auto Range = GlobalMMapInfo.equal_range(NameToUse);
2029 for (auto I = Range.first; I != Range.second; ++I) {
2030 MMapInfo &MMapInfo = I->second;
2031 if (BC->HasFixedLoadAddress && MMapInfo.MMapAddress) {
2032 // Check that the binary mapping matches one of the segments.
2033 bool MatchFound = llvm::any_of(
2034 llvm::make_second_range(BC->SegmentMapInfo),
2035 [&](SegmentInfo &SegInfo) {
2036 // The mapping is page-aligned and hence the MMapAddress could be
2037 // different from the segment start address. We cannot know the page
2038 // size of the mapping, but we know it should not exceed the segment
2039 // alignment value. Hence we are performing an approximate check.
2040 return SegInfo.Address >= MMapInfo.MMapAddress &&
2041 SegInfo.Address - MMapInfo.MMapAddress < SegInfo.Alignment;
2042 });
2043 if (!MatchFound) {
2044 errs() << "PERF2BOLT-WARNING: ignoring mapping of " << NameToUse
2045 << " at 0x" << Twine::utohexstr(MMapInfo.MMapAddress) << '\n';
2046 continue;
2047 }
2048 }
2049
2050 // Set base address for shared objects.
2051 if (!BC->HasFixedLoadAddress) {
2052 Optional<uint64_t> BaseAddress =
2053 BC->getBaseAddressForMapping(MMapInfo.MMapAddress, MMapInfo.Offset);
2054 if (!BaseAddress) {
2055 errs() << "PERF2BOLT-WARNING: unable to find base address of the "
2056 "binary when memory mapped at 0x"
2057 << Twine::utohexstr(MMapInfo.MMapAddress)
2058 << " using file offset 0x" << Twine::utohexstr(MMapInfo.Offset)
2059 << ". Ignoring profile data for this mapping\n";
2060 continue;
2061 } else {
2062 MMapInfo.BaseAddress = *BaseAddress;
2063 }
2064 }
2065
2066 BinaryMMapInfo.insert(std::make_pair(MMapInfo.PID, MMapInfo));
2067 }
2068
2069 if (BinaryMMapInfo.empty()) {
2070 if (errs().has_colors())
2071 errs().changeColor(raw_ostream::RED);
2072 errs() << "PERF2BOLT-ERROR: could not find a profile matching binary \""
2073 << BC->getFilename() << "\".";
2074 if (!GlobalMMapInfo.empty()) {
2075 errs() << " Profile for the following binary name(s) is available:\n";
2076 for (auto I = GlobalMMapInfo.begin(), IE = GlobalMMapInfo.end(); I != IE;
2077 I = GlobalMMapInfo.upper_bound(I->first))
2078 errs() << " " << I->first << '\n';
2079 errs() << "Please rename the input binary.\n";
2080 } else {
2081 errs() << " Failed to extract any binary name from a profile.\n";
2082 }
2083 if (errs().has_colors())
2084 errs().resetColor();
2085
2086 exit(1);
2087 }
2088
2089 return std::error_code();
2090}
2091
2092std::error_code DataAggregator::parseTaskEvents() {
2093 outs() << "PERF2BOLT: parsing perf-script task events output\n";
2094 NamedRegionTimer T("parseTaskEvents", "Parsing task events", TimerGroupName,
2095 TimerGroupDesc, opts::TimeAggregator);
2096
2097 while (hasData()) {
2098 if (Optional<int32_t> CommInfo = parseCommExecEvent()) {
2099 // Remove forked child that ran execve
2100 auto MMapInfoIter = BinaryMMapInfo.find(*CommInfo);
2101 if (MMapInfoIter != BinaryMMapInfo.end() && MMapInfoIter->second.Forked)
2102 BinaryMMapInfo.erase(MMapInfoIter);
2103 consumeRestOfLine();
2104 continue;
2105 }
2106
2107 Optional<ForkInfo> ForkInfo = parseForkEvent();
2108 if (!ForkInfo)
2109 continue;
2110
2111 if (ForkInfo->ParentPID == ForkInfo->ChildPID)
2112 continue;
2113
2114 if (ForkInfo->Time == 0) {
2115 // Process was forked and mmaped before perf ran. In this case the child
2116 // should have its own mmap entry unless it was execve'd.
2117 continue;
2118 }
2119
2120 auto MMapInfoIter = BinaryMMapInfo.find(ForkInfo->ParentPID);
2121 if (MMapInfoIter == BinaryMMapInfo.end())
2122 continue;
2123
2124 MMapInfo MMapInfo = MMapInfoIter->second;
2125 MMapInfo.PID = ForkInfo->ChildPID;
2126 MMapInfo.Forked = true;
2127 BinaryMMapInfo.insert(std::make_pair(MMapInfo.PID, MMapInfo));
2128 }
2129
2130 outs() << "PERF2BOLT: input binary is associated with "
2131 << BinaryMMapInfo.size() << " PID(s)\n";
2132
2133 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { for (std::pair<const uint64_t, MMapInfo
> &MMI : BinaryMMapInfo) outs() << " " <<
MMI.second.PID << (MMI.second.Forked ? " (forked)" : ""
) << ": (0x" << Twine::utohexstr(MMI.second.MMapAddress
) << ": 0x" << Twine::utohexstr(MMI.second.Size) <<
")\n"; }; } } while (false)
2134 for (std::pair<const uint64_t, MMapInfo> &MMI : BinaryMMapInfo)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { for (std::pair<const uint64_t, MMapInfo
> &MMI : BinaryMMapInfo) outs() << " " <<
MMI.second.PID << (MMI.second.Forked ? " (forked)" : ""
) << ": (0x" << Twine::utohexstr(MMI.second.MMapAddress
) << ": 0x" << Twine::utohexstr(MMI.second.Size) <<
")\n"; }; } } while (false)
2135 outs() << " " << MMI.second.PID << (MMI.second.Forked ? " (forked)" : "")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { for (std::pair<const uint64_t, MMapInfo
> &MMI : BinaryMMapInfo) outs() << " " <<
MMI.second.PID << (MMI.second.Forked ? " (forked)" : ""
) << ": (0x" << Twine::utohexstr(MMI.second.MMapAddress
) << ": 0x" << Twine::utohexstr(MMI.second.Size) <<
")\n"; }; } } while (false)
2136 << ": (0x" << Twine::utohexstr(MMI.second.MMapAddress) << ": 0x"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { for (std::pair<const uint64_t, MMapInfo
> &MMI : BinaryMMapInfo) outs() << " " <<
MMI.second.PID << (MMI.second.Forked ? " (forked)" : ""
) << ": (0x" << Twine::utohexstr(MMI.second.MMapAddress
) << ": 0x" << Twine::utohexstr(MMI.second.Size) <<
")\n"; }; } } while (false)
2137 << Twine::utohexstr(MMI.second.Size) << ")\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { for (std::pair<const uint64_t, MMapInfo
> &MMI : BinaryMMapInfo) outs() << " " <<
MMI.second.PID << (MMI.second.Forked ? " (forked)" : ""
) << ": (0x" << Twine::utohexstr(MMI.second.MMapAddress
) << ": 0x" << Twine::utohexstr(MMI.second.Size) <<
")\n"; }; } } while (false)
2138 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aggregator")) { { for (std::pair<const uint64_t, MMapInfo
> &MMI : BinaryMMapInfo) outs() << " " <<
MMI.second.PID << (MMI.second.Forked ? " (forked)" : ""
) << ": (0x" << Twine::utohexstr(MMI.second.MMapAddress
) << ": 0x" << Twine::utohexstr(MMI.second.Size) <<
")\n"; }; } } while (false)
;
2139
2140 return std::error_code();
2141}
2142
2143Optional<std::pair<StringRef, StringRef>>
2144DataAggregator::parseNameBuildIDPair() {
2145 while (checkAndConsumeFS()) {
2146 }
2147
2148 ErrorOr<StringRef> BuildIDStr = parseString(FieldSeparator, true);
2149 if (std::error_code EC = BuildIDStr.getError())
2150 return std::nullopt;
2151
2152 // If one of the strings is missing, don't issue a parsing error, but still
2153 // do not return a value.
2154 consumeAllRemainingFS();
2155 if (checkNewLine())
2156 return std::nullopt;
2157
2158 ErrorOr<StringRef> NameStr = parseString(FieldSeparator, true);
2159 if (std::error_code EC = NameStr.getError())
2160 return std::nullopt;
2161
2162 consumeRestOfLine();
2163 return std::make_pair(NameStr.get(), BuildIDStr.get());
2164}
2165
2166bool DataAggregator::hasAllBuildIDs() {
2167 const StringRef SavedParsingBuf = ParsingBuf;
2168
2169 if (!hasData())
2170 return false;
2171
2172 bool HasInvalidEntries = false;
2173 while (hasData()) {
2174 if (!parseNameBuildIDPair()) {
2175 HasInvalidEntries = true;
2176 break;
2177 }
2178 }
2179
2180 ParsingBuf = SavedParsingBuf;
2181
2182 return !HasInvalidEntries;
2183}
2184
2185Optional<StringRef>
2186DataAggregator::getFileNameForBuildID(StringRef FileBuildID) {
2187 const StringRef SavedParsingBuf = ParsingBuf;
2188
2189 StringRef FileName;
2190 while (hasData()) {
2191 Optional<std::pair<StringRef, StringRef>> IDPair = parseNameBuildIDPair();
2192 if (!IDPair) {
2193 consumeRestOfLine();
2194 continue;
2195 }
2196
2197 if (IDPair->second.startswith(FileBuildID)) {
2198 FileName = sys::path::filename(IDPair->first);
2199 break;
2200 }
2201 }
2202
2203 ParsingBuf = SavedParsingBuf;
2204
2205 if (!FileName.empty())
2206 return FileName;
2207
2208 return std::nullopt;
2209}
2210
2211std::error_code
2212DataAggregator::writeAggregatedFile(StringRef OutputFilename) const {
2213 std::error_code EC;
2214 raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None);
2215 if (EC)
2216 return EC;
2217
2218 bool WriteMemLocs = false;
2219
2220 auto writeLocation = [&OutFile, &WriteMemLocs](const Location &Loc) {
2221 if (WriteMemLocs)
2222 OutFile << (Loc.IsSymbol ? "4 " : "3 ");
2223 else
2224 OutFile << (Loc.IsSymbol ? "1 " : "0 ");
2225 OutFile << (Loc.Name.empty() ? "[unknown]" : getEscapedName(Loc.Name))
2226 << " " << Twine::utohexstr(Loc.Offset) << FieldSeparator;
2227 };
2228
2229 uint64_t BranchValues = 0;
2230 uint64_t MemValues = 0;
2231
2232 if (BAT)
2233 OutFile << "boltedcollection\n";
2234 if (opts::BasicAggregation) {
2235 OutFile << "no_lbr";
2236 for (const StringMapEntry<std::nullopt_t> &Entry : EventNames)
2237 OutFile << " " << Entry.getKey();
2238 OutFile << "\n";
2239
2240 for (const StringMapEntry<FuncSampleData> &Func : NamesToSamples) {
2241 for (const SampleInfo &SI : Func.getValue().Data) {
2242 writeLocation(SI.Loc);
2243 OutFile << SI.Hits << "\n";
2244 ++BranchValues;
2245 }
2246 }
2247 } else {
2248 for (const StringMapEntry<FuncBranchData> &Func : NamesToBranches) {
2249 for (const llvm::bolt::BranchInfo &BI : Func.getValue().Data) {
2250 writeLocation(BI.From);
2251 writeLocation(BI.To);
2252 OutFile << BI.Mispreds << " " << BI.Branches << "\n";
2253 ++BranchValues;
2254 }
2255 for (const llvm::bolt::BranchInfo &BI : Func.getValue().EntryData) {
2256 // Do not output if source is a known symbol, since this was already
2257 // accounted for in the source function
2258 if (BI.From.IsSymbol)
2259 continue;
2260 writeLocation(BI.From);
2261 writeLocation(BI.To);
2262 OutFile << BI.Mispreds << " " << BI.Branches << "\n";
2263 ++BranchValues;
2264 }
2265 }
2266
2267 WriteMemLocs = true;
2268 for (const StringMapEntry<FuncMemData> &Func : NamesToMemEvents) {
2269 for (const MemInfo &MemEvent : Func.getValue().Data) {
2270 writeLocation(MemEvent.Offset);
2271 writeLocation(MemEvent.Addr);
2272 OutFile << MemEvent.Count << "\n";
2273 ++MemValues;
2274 }
2275 }
2276 }
2277
2278 outs() << "PERF2BOLT: wrote " << BranchValues << " objects and " << MemValues
2279 << " memory objects to " << OutputFilename << "\n";
2280
2281 return std::error_code();
2282}
2283
2284void DataAggregator::dump() const { DataReader::dump(); }
2285
2286void DataAggregator::dump(const LBREntry &LBR) const {
2287 Diag << "From: " << Twine::utohexstr(LBR.From)
2288 << " To: " << Twine::utohexstr(LBR.To) << " Mispred? " << LBR.Mispred
2289 << "\n";
2290}
2291
2292void DataAggregator::dump(const PerfBranchSample &Sample) const {
2293 Diag << "Sample LBR entries: " << Sample.LBR.size() << "\n";
2294 for (const LBREntry &LBR : Sample.LBR)
2295 dump(LBR);
2296}
2297
2298void DataAggregator::dump(const PerfMemSample &Sample) const {
2299 Diag << "Sample mem entries: " << Sample.PC << ": " << Sample.Addr << "\n";
2300}