Line data Source code
1 : //===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file defines a JITEventListener object that tells perf about JITted
11 : // functions, including source line information.
12 : //
13 : // Documentation for perf jit integration is available at:
14 : // https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt
15 : // https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt
16 : //
17 : //===----------------------------------------------------------------------===//
18 :
19 : #include "llvm/ADT/Twine.h"
20 : #include "llvm/Config/config.h"
21 : #include "llvm/DebugInfo/DWARF/DWARFContext.h"
22 : #include "llvm/ExecutionEngine/JITEventListener.h"
23 : #include "llvm/Object/ObjectFile.h"
24 : #include "llvm/Object/SymbolSize.h"
25 : #include "llvm/Support/Debug.h"
26 : #include "llvm/Support/Errno.h"
27 : #include "llvm/Support/FileSystem.h"
28 : #include "llvm/Support/MemoryBuffer.h"
29 : #include "llvm/Support/Mutex.h"
30 : #include "llvm/Support/MutexGuard.h"
31 : #include "llvm/Support/Path.h"
32 : #include "llvm/Support/Process.h"
33 : #include "llvm/Support/Threading.h"
34 : #include "llvm/Support/raw_ostream.h"
35 :
36 : #include <sys/mman.h> // mmap()
37 : #include <sys/types.h> // getpid()
38 : #include <time.h> // clock_gettime(), time(), localtime_r() */
39 : #include <unistd.h> // for getpid(), read(), close()
40 :
41 : using namespace llvm;
42 : using namespace llvm::object;
43 : typedef DILineInfoSpecifier::FileLineInfoKind FileLineInfoKind;
44 :
45 : namespace {
46 :
47 : // language identifier (XXX: should we generate something better from debug
48 : // info?)
49 : #define JIT_LANG "llvm-IR"
50 : #define LLVM_PERF_JIT_MAGIC \
51 : ((uint32_t)'J' << 24 | (uint32_t)'i' << 16 | (uint32_t)'T' << 8 | \
52 : (uint32_t)'D')
53 : #define LLVM_PERF_JIT_VERSION 1
54 :
55 : // bit 0: set if the jitdump file is using an architecture-specific timestamp
56 : // clock source
57 : #define JITDUMP_FLAGS_ARCH_TIMESTAMP (1ULL << 0)
58 :
59 : struct LLVMPerfJitHeader;
60 :
61 : class PerfJITEventListener : public JITEventListener {
62 : public:
63 : PerfJITEventListener();
64 0 : ~PerfJITEventListener() {
65 0 : if (MarkerAddr)
66 0 : CloseMarker();
67 0 : }
68 0 :
69 : void NotifyObjectEmitted(const ObjectFile &Obj,
70 : const RuntimeDyld::LoadedObjectInfo &L) override;
71 0 : void NotifyFreeingObject(const ObjectFile &Obj) override;
72 0 :
73 0 : private:
74 0 : bool InitDebuggingDir();
75 0 : bool OpenMarker();
76 : void CloseMarker();
77 : static bool FillMachine(LLVMPerfJitHeader &hdr);
78 :
79 : void NotifyCode(Expected<llvm::StringRef> &Symbol, uint64_t CodeAddr,
80 : uint64_t CodeSize);
81 : void NotifyDebug(uint64_t CodeAddr, DILineInfoTable Lines);
82 :
83 : // cache lookups
84 : pid_t Pid;
85 :
86 : // base directory for output data
87 : std::string JitPath;
88 :
89 : // output data stream, closed via Dumpstream
90 : int DumpFd = -1;
91 :
92 : // output data stream
93 : std::unique_ptr<raw_fd_ostream> Dumpstream;
94 :
95 : // prevent concurrent dumps from messing up the output file
96 : sys::Mutex Mutex;
97 :
98 : // perf mmap marker
99 : void *MarkerAddr = NULL;
100 :
101 : // perf support ready
102 : bool SuccessfullyInitialized = false;
103 :
104 : // identifier for functions, primarily to identify when moving them around
105 : uint64_t CodeGeneration = 1;
106 : };
107 :
108 : // The following are POD struct definitions from the perf jit specification
109 :
110 : enum LLVMPerfJitRecordType {
111 : JIT_CODE_LOAD = 0,
112 : JIT_CODE_MOVE = 1, // not emitted, code isn't moved
113 : JIT_CODE_DEBUG_INFO = 2,
114 : JIT_CODE_CLOSE = 3, // not emitted, unnecessary
115 : JIT_CODE_UNWINDING_INFO = 4, // not emitted
116 :
117 : JIT_CODE_MAX
118 : };
119 :
120 : struct LLVMPerfJitHeader {
121 : uint32_t Magic; // characters "JiTD"
122 : uint32_t Version; // header version
123 : uint32_t TotalSize; // total size of header
124 : uint32_t ElfMach; // elf mach target
125 : uint32_t Pad1; // reserved
126 : uint32_t Pid;
127 : uint64_t Timestamp; // timestamp
128 : uint64_t Flags; // flags
129 : };
130 :
131 : // record prefix (mandatory in each record)
132 : struct LLVMPerfJitRecordPrefix {
133 : uint32_t Id; // record type identifier
134 : uint32_t TotalSize;
135 : uint64_t Timestamp;
136 : };
137 :
138 : struct LLVMPerfJitRecordCodeLoad {
139 : LLVMPerfJitRecordPrefix Prefix;
140 :
141 : uint32_t Pid;
142 : uint32_t Tid;
143 : uint64_t Vma;
144 : uint64_t CodeAddr;
145 : uint64_t CodeSize;
146 : uint64_t CodeIndex;
147 : };
148 :
149 : struct LLVMPerfJitDebugEntry {
150 : uint64_t Addr;
151 : int Lineno; // source line number starting at 1
152 : int Discrim; // column discriminator, 0 is default
153 : // followed by null terminated filename, \xff\0 if same as previous entry
154 : };
155 :
156 : struct LLVMPerfJitRecordDebugInfo {
157 : LLVMPerfJitRecordPrefix Prefix;
158 :
159 : uint64_t CodeAddr;
160 : uint64_t NrEntry;
161 : // followed by NrEntry LLVMPerfJitDebugEntry records
162 : };
163 :
164 : static inline uint64_t timespec_to_ns(const struct timespec *ts) {
165 : const uint64_t NanoSecPerSec = 1000000000;
166 : return ((uint64_t)ts->tv_sec * NanoSecPerSec) + ts->tv_nsec;
167 : }
168 :
169 : static inline uint64_t perf_get_timestamp(void) {
170 : struct timespec ts;
171 : int ret;
172 0 :
173 : ret = clock_gettime(CLOCK_MONOTONIC, &ts);
174 155 : if (ret)
175 : return 0;
176 :
177 155 : return timespec_to_ns(&ts);
178 : }
179 :
180 : PerfJITEventListener::PerfJITEventListener() : Pid(::getpid()) {
181 155 : // check if clock-source is supported
182 155 : if (!perf_get_timestamp()) {
183 : errs() << "kernel does not support CLOCK_MONOTONIC\n";
184 : return;
185 155 : }
186 :
187 : if (!InitDebuggingDir()) {
188 155 : errs() << "could not initialize debugging directory\n";
189 : return;
190 155 : }
191 0 :
192 155 : std::string Filename;
193 : raw_string_ostream FilenameBuf(Filename);
194 : FilenameBuf << JitPath << "/jit-" << Pid << ".dump";
195 155 :
196 155 : // Need to open ourselves, because we need to hand the FD to OpenMarker() and
197 155 : // raw_fd_ostream doesn't expose the FD.
198 : using sys::fs::openFileForWrite;
199 : if (auto EC =
200 : openFileForReadWrite(FilenameBuf.str(), DumpFd,
201 0 : sys::fs::CD_CreateNew, sys::fs::OF_None)) {
202 0 : errs() << "could not open JIT dump file " << FilenameBuf.str() << ": "
203 : << EC.message() << "\n";
204 : return;
205 : }
206 :
207 0 : Dumpstream = make_unique<raw_fd_ostream>(DumpFd, true);
208 0 :
209 0 : LLVMPerfJitHeader Header = {0};
210 0 : if (!FillMachine(Header))
211 0 : return;
212 :
213 : // signal this process emits JIT information
214 : if (!OpenMarker())
215 0 : return;
216 :
217 0 : // emit dumpstream header
218 0 : Header.Magic = LLVM_PERF_JIT_MAGIC;
219 : Header.Version = LLVM_PERF_JIT_VERSION;
220 : Header.TotalSize = sizeof(Header);
221 : Header.Pid = Pid;
222 0 : Header.Timestamp = perf_get_timestamp();
223 : Dumpstream->write(reinterpret_cast<const char *>(&Header), sizeof(Header));
224 :
225 : // Everything initialized, can do profiling now.
226 0 : if (!Dumpstream->has_error())
227 0 : SuccessfullyInitialized = true;
228 0 : }
229 0 :
230 0 : void PerfJITEventListener::NotifyObjectEmitted(
231 0 : const ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &L) {
232 :
233 : if (!SuccessfullyInitialized)
234 0 : return;
235 0 :
236 : OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj);
237 : const ObjectFile &DebugObj = *DebugObjOwner.getBinary();
238 83 :
239 : // Get the address of the object image for use as a unique identifier
240 : std::unique_ptr<DIContext> Context = DWARFContext::create(DebugObj);
241 83 :
242 83 : // Use symbol info to iterate over functions in the object.
243 : for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) {
244 0 : SymbolRef Sym = P.first;
245 : std::string SourceFileName;
246 :
247 : Expected<SymbolRef::Type> SymTypeOrErr = Sym.getType();
248 0 : if (!SymTypeOrErr) {
249 : // There's not much we can with errors here
250 : consumeError(SymTypeOrErr.takeError());
251 0 : continue;
252 0 : }
253 : SymbolRef::Type SymType = *SymTypeOrErr;
254 : if (SymType != SymbolRef::ST_Function)
255 : continue;
256 0 :
257 : Expected<StringRef> Name = Sym.getName();
258 0 : if (!Name) {
259 0 : consumeError(Name.takeError());
260 : continue;
261 0 : }
262 0 :
263 : Expected<uint64_t> AddrOrErr = Sym.getAddress();
264 : if (!AddrOrErr) {
265 : consumeError(AddrOrErr.takeError());
266 0 : continue;
267 0 : }
268 0 : uint64_t Addr = *AddrOrErr;
269 : uint64_t Size = P.second;
270 :
271 : // According to spec debugging info has to come before loading the
272 0 : // corresonding code load.
273 0 : DILineInfoTable Lines = Context->getLineInfoForAddressRange(
274 : Addr, Size, FileLineInfoKind::AbsoluteFilePath);
275 :
276 0 : NotifyDebug(Addr, Lines);
277 0 : NotifyCode(Name, Addr, Size);
278 : }
279 :
280 : Dumpstream->flush();
281 : }
282 0 :
283 : void PerfJITEventListener::NotifyFreeingObject(const ObjectFile &Obj) {
284 0 : // perf currently doesn't have an interface for unloading. But munmap()ing the
285 0 : // code section does, so that's ok.
286 : }
287 :
288 0 : bool PerfJITEventListener::InitDebuggingDir() {
289 : time_t Time;
290 : struct tm LocalTime;
291 0 : char TimeBuffer[sizeof("YYYYMMDD")];
292 : SmallString<64> Path;
293 :
294 0 : // search for location to dump data to
295 : if (const char *BaseDir = getenv("JITDUMPDIR"))
296 155 : Path.append(BaseDir);
297 : else if (!sys::path::home_directory(Path))
298 : Path = ".";
299 :
300 : // create debug directory
301 : Path += "/.debug/jit/";
302 : if (auto EC = sys::fs::create_directories(Path)) {
303 155 : errs() << "could not create jit cache directory " << Path << ": "
304 : << EC.message() << "\n";
305 155 : return false;
306 : }
307 :
308 : // create unique directory for dump data related to this process
309 : time(&Time);
310 155 : localtime_r(&Time, &LocalTime);
311 310 : strftime(TimeBuffer, sizeof(TimeBuffer), "%Y%m%d", &LocalTime);
312 155 : Path += JIT_LANG "-jit-";
313 155 : Path += TimeBuffer;
314 :
315 : SmallString<128> UniqueDebugDir;
316 :
317 0 : using sys::fs::createUniqueDirectory;
318 0 : if (auto EC = createUniqueDirectory(Path, UniqueDebugDir)) {
319 0 : errs() << "could not create unique jit cache directory " << UniqueDebugDir
320 : << ": " << EC.message() << "\n";
321 : return false;
322 : }
323 :
324 : JitPath = UniqueDebugDir.str();
325 :
326 0 : return true;
327 0 : }
328 0 :
329 0 : bool PerfJITEventListener::OpenMarker() {
330 : // We mmap the jitdump to create an MMAP RECORD in perf.data file. The mmap
331 : // is captured either live (perf record running when we mmap) or in deferred
332 0 : // mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump
333 : // file for more meta data info about the jitted code. Perf report/annotate
334 0 : // detect this special filename and process the jitdump file.
335 : //
336 : // Mapping must be PROT_EXEC to ensure it is captured by perf record
337 0 : // even when not using -d option.
338 : MarkerAddr = ::mmap(NULL, sys::Process::getPageSize(), PROT_READ | PROT_EXEC,
339 : MAP_PRIVATE, DumpFd, 0);
340 :
341 : if (MarkerAddr == MAP_FAILED) {
342 : errs() << "could not mmap JIT marker\n";
343 : return false;
344 : }
345 : return true;
346 0 : }
347 :
348 : void PerfJITEventListener::CloseMarker() {
349 0 : if (!MarkerAddr)
350 0 : return;
351 0 :
352 : munmap(MarkerAddr, sys::Process::getPageSize());
353 : MarkerAddr = nullptr;
354 : }
355 :
356 0 : bool PerfJITEventListener::FillMachine(LLVMPerfJitHeader &hdr) {
357 0 : char id[16];
358 0 : struct {
359 : uint16_t e_type;
360 0 : uint16_t e_machine;
361 0 : } info;
362 :
363 : size_t RequiredMemory = sizeof(id) + sizeof(info);
364 0 :
365 : ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
366 : MemoryBuffer::getFileSlice("/proc/self/exe",
367 : RequiredMemory,
368 : 0);
369 :
370 : // This'll not guarantee that enough data was actually read from the
371 : // underlying file. Instead the trailing part of the buffer would be
372 : // zeroed. Given the ELF signature check below that seems ok though,
373 : // it's unlikely that the file ends just after that, and the
374 : // consequence would just be that perf wouldn't recognize the
375 : // signature.
376 0 : if (auto EC = MB.getError()) {
377 : errs() << "could not open /proc/self/exe: " << EC.message() << "\n";
378 : return false;
379 : }
380 :
381 : memcpy(&id, (*MB)->getBufferStart(), sizeof(id));
382 : memcpy(&info, (*MB)->getBufferStart() + sizeof(id), sizeof(info));
383 :
384 0 : // check ELF signature
385 0 : if (id[0] != 0x7f || id[1] != 'E' || id[2] != 'L' || id[3] != 'F') {
386 0 : errs() << "invalid elf signature\n";
387 : return false;
388 : }
389 0 :
390 0 : hdr.ElfMach = info.e_machine;
391 :
392 : return true;
393 0 : }
394 0 :
395 0 : void PerfJITEventListener::NotifyCode(Expected<llvm::StringRef> &Symbol,
396 : uint64_t CodeAddr, uint64_t CodeSize) {
397 : assert(SuccessfullyInitialized);
398 0 :
399 : // 0 length functions can't have samples.
400 0 : if (CodeSize == 0)
401 : return;
402 :
403 0 : LLVMPerfJitRecordCodeLoad rec;
404 : rec.Prefix.Id = JIT_CODE_LOAD;
405 : rec.Prefix.TotalSize = sizeof(rec) + // debug record itself
406 : Symbol->size() + 1 + // symbol name
407 : CodeSize; // and code
408 0 : rec.Prefix.Timestamp = perf_get_timestamp();
409 0 :
410 : rec.CodeSize = CodeSize;
411 : rec.Vma = 0;
412 0 : rec.CodeAddr = CodeAddr;
413 0 : rec.Pid = Pid;
414 0 : rec.Tid = get_threadid();
415 :
416 0 : // avoid interspersing output
417 : MutexGuard Guard(Mutex);
418 0 :
419 0 : rec.CodeIndex = CodeGeneration++; // under lock!
420 0 :
421 0 : Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec));
422 0 : Dumpstream->write(Symbol->data(), Symbol->size() + 1);
423 : Dumpstream->write(reinterpret_cast<const char *>(CodeAddr), CodeSize);
424 : }
425 :
426 : void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr,
427 0 : DILineInfoTable Lines) {
428 : assert(SuccessfullyInitialized);
429 0 :
430 0 : // Didn't get useful debug info.
431 0 : if (Lines.empty())
432 : return;
433 :
434 0 : LLVMPerfJitRecordDebugInfo rec;
435 : rec.Prefix.Id = JIT_CODE_DEBUG_INFO;
436 : rec.Prefix.TotalSize = sizeof(rec); // will be increased further
437 : rec.Prefix.Timestamp = perf_get_timestamp();
438 : rec.CodeAddr = CodeAddr;
439 0 : rec.NrEntry = Lines.size();
440 0 :
441 : // compute total size size of record (variable due to filenames)
442 : DILineInfoTable::iterator Begin = Lines.begin();
443 0 : DILineInfoTable::iterator End = Lines.end();
444 0 : for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
445 0 : DILineInfo &line = It->second;
446 0 : rec.Prefix.TotalSize += sizeof(LLVMPerfJitDebugEntry);
447 0 : rec.Prefix.TotalSize += line.FileName.size() + 1;
448 : }
449 :
450 : // The debug_entry describes the source line information. It is defined as
451 : // follows in order:
452 0 : // * uint64_t code_addr: address of function for which the debug information
453 : // is generated
454 0 : // * uint32_t line : source file line number (starting at 1)
455 0 : // * uint32_t discrim : column discriminator, 0 is default
456 : // * char name[n] : source file name in ASCII, including null termination
457 :
458 : // avoid interspersing output
459 : MutexGuard Guard(Mutex);
460 :
461 : Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec));
462 :
463 : for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
464 : LLVMPerfJitDebugEntry LineInfo;
465 : DILineInfo &Line = It->second;
466 :
467 : LineInfo.Addr = It->first;
468 : // The function re-created by perf is preceded by a elf
469 0 : // header. Need to adjust for that, otherwise the results are
470 : // wrong.
471 0 : LineInfo.Addr += 0x40;
472 : LineInfo.Lineno = Line.Line;
473 : LineInfo.Discrim = Line.Discriminator;
474 :
475 0 : Dumpstream->write(reinterpret_cast<const char *>(&LineInfo),
476 : sizeof(LineInfo));
477 : Dumpstream->write(Line.FileName.c_str(), Line.FileName.size() + 1);
478 : }
479 0 : }
480 0 :
481 0 : // There should be only a single event listener per process, otherwise perf gets
482 : // confused.
483 : llvm::ManagedStatic<PerfJITEventListener> PerfListener;
484 0 :
485 0 : } // end anonymous namespace
486 :
487 : namespace llvm {
488 : JITEventListener *JITEventListener::createPerfJITEventListener() {
489 : return &*PerfListener;
490 : }
491 :
492 : } // namespace llvm
493 :
494 : LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void)
495 : {
496 155 : return wrap(JITEventListener::createPerfJITEventListener());
497 155 : }
|
