LLVM 19.0.0git
InstrProfData.inc
Go to the documentation of this file.
1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7\*===----------------------------------------------------------------------===*/
8/*
9 * This is the main file that defines all the data structure, signature,
10 * constant literals that are shared across profiling runtime library,
11 * compiler (instrumentation), and host tools (reader/writer). The entities
12 * defined in this file affect the profile runtime ABI, the raw profile format,
13 * or both.
14 *
15 * The file has two identical copies. The primary copy lives in LLVM and
16 * the other one sits in compiler-rt/lib/profile directory. To make changes
17 * in this file, first modify the primary copy and copy it over to compiler-rt.
18 * Testing of any change in this file can start only after the two copies are
19 * synced up.
20 *
21 * The first part of the file includes macros that defines types, names, and
22 * initializers for the member fields of the core data structures. The field
23 * declarations for one structure is enabled by defining the field activation
24 * macro associated with that structure. Only one field activation record
25 * can be defined at one time and the rest definitions will be filtered out by
26 * the preprocessor.
27 *
28 * Examples of how the template is used to instantiate structure definition:
29 * 1. To declare a structure:
30 *
31 * struct ProfData {
32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33 * Type Name;
34 * #include "llvm/ProfileData/InstrProfData.inc"
35 * };
36 *
37 * 2. To construct LLVM type arrays for the struct type:
38 *
39 * Type *DataTypes[] = {
40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41 * LLVMType,
42 * #include "llvm/ProfileData/InstrProfData.inc"
43 * };
44 *
45 * 4. To construct constant array for the initializers:
46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47 * Initializer,
48 * Constant *ConstantVals[] = {
49 * #include "llvm/ProfileData/InstrProfData.inc"
50 * };
51 *
52 *
53 * The second part of the file includes definitions all other entities that
54 * are related to runtime ABI and format. When no field activation macro is
55 * defined, this file can be included to introduce the definitions.
56 *
57\*===----------------------------------------------------------------------===*/
58
59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60 * the compiler runtime. */
61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
63#endif
64
65// clang-format off:consider re-enabling clang-format if auto-formatted C macros
66// are readable (e.g., after `issue #82426` is fixed)
67/* INSTR_PROF_DATA start. */
68/* Definition of member fields of the per-function control structure. */
69#ifndef INSTR_PROF_DATA
70#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
71#else
72#define INSTR_PROF_DATA_DEFINED
73#endif
75 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
76 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
77INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
78 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
79 Inc->getHash()->getZExtValue()))
80INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
81INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)
82/* This is used to map function pointers for the indirect call targets to
83 * function name hashes during the conversion from raw to merged profile
84 * data.
85 */
86INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \
87 FunctionAddr)
88INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \
89 ValuesPtrExpr)
90INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
91 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
92INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
93 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
94INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
95 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
96#undef INSTR_PROF_DATA
97/* INSTR_PROF_DATA end. */
98
99
100/* This is an internal data structure used by value profiler. It
101 * is defined here to allow serialization code sharing by LLVM
102 * to be used in unit test.
103 *
104 * typedef struct ValueProfNode {
105 * // InstrProfValueData VData;
106 * uint64_t Value;
107 * uint64_t Count;
108 * struct ValueProfNode *Next;
109 * } ValueProfNode;
110 */
111/* INSTR_PROF_VALUE_NODE start. */
112#ifndef INSTR_PROF_VALUE_NODE
113#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
114#else
115#define INSTR_PROF_DATA_DEFINED
116#endif
118 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
119INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
120 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
121INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \
122 ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
123#undef INSTR_PROF_VALUE_NODE
124/* INSTR_PROF_VALUE_NODE end. */
125
126/* INSTR_PROF_RAW_HEADER start */
127/* Definition of member fields of the raw profile header data structure. */
128/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating
129 raw profile format. */
130#ifndef INSTR_PROF_RAW_HEADER
131#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
132#else
133#define INSTR_PROF_DATA_DEFINED
134#endif
135INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
136INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
137INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
138INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)
139INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
140INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)
141INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
142INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)
143INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)
144INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
145INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
146 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
148 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
149INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
150INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
151#undef INSTR_PROF_RAW_HEADER
152/* INSTR_PROF_RAW_HEADER end */
153
154/* VALUE_PROF_FUNC_PARAM start */
155/* Definition of parameter types of the runtime API used to do value profiling
156 * for a given value site.
157 */
158#ifndef VALUE_PROF_FUNC_PARAM
159#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
160#define INSTR_PROF_COMMA
161#else
162#define INSTR_PROF_DATA_DEFINED
163#define INSTR_PROF_COMMA ,
164#endif
165VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
166 INSTR_PROF_COMMA
167VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA
168VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
169#undef VALUE_PROF_FUNC_PARAM
170#undef INSTR_PROF_COMMA
171/* VALUE_PROF_FUNC_PARAM end */
172
173/* VALUE_PROF_KIND start */
174#ifndef VALUE_PROF_KIND
175#define VALUE_PROF_KIND(Enumerator, Value, Descr)
176#else
177#define INSTR_PROF_DATA_DEFINED
178#endif
179/* For indirect function call value profiling, the addresses of the target
180 * functions are profiled by the instrumented code. The target addresses are
181 * written in the raw profile data and converted to target function name's MD5
182 * hash by the profile reader during deserialization. Typically, this happens
183 * when the raw profile data is read during profile merging.
184 *
185 * For this remapping the ProfData is used. ProfData contains both the function
186 * name hash and the function address.
187 */
188VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
189/* For memory intrinsic functions size profiling. */
190VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
191/* These two kinds must be the last to be
192 * declared. This is to make sure the string
193 * array created with the template can be
194 * indexed with the kind value.
195 */
196VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
197VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
198
199#undef VALUE_PROF_KIND
200/* VALUE_PROF_KIND end */
201
202#undef COVMAP_V2_OR_V3
203#ifdef COVMAP_V2
204#define COVMAP_V2_OR_V3
205#endif
206#ifdef COVMAP_V3
207#define COVMAP_V2_OR_V3
208#endif
209
210/* COVMAP_FUNC_RECORD start */
211/* Definition of member fields of the function record structure in coverage
212 * map.
213 */
214#ifndef COVMAP_FUNC_RECORD
215#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
216#else
217#define INSTR_PROF_DATA_DEFINED
218#endif
219#ifdef COVMAP_V1
221 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
224 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
225 NameValue.size()))
226#endif
227#ifdef COVMAP_V2_OR_V3
228COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
229 llvm::ConstantInt::get( \
230 llvm::Type::getInt64Ty(Ctx), NameHash))
231#endif
233 llvm::ConstantInt::get( \
234 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
235COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
236 llvm::ConstantInt::get( \
237 llvm::Type::getInt64Ty(Ctx), FuncHash))
238#ifdef COVMAP_V3
239COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
240 llvm::ConstantInt::get( \
241 llvm::Type::getInt64Ty(Ctx), FilenamesRef))
242COVMAP_FUNC_RECORD(const char, \
244 CoverageMapping.size()), \
245 CoverageMapping,
247 CoverageMapping, CoverageMapping.size(), \
249#endif
250#undef COVMAP_FUNC_RECORD
251/* COVMAP_FUNC_RECORD end. */
252
253/* COVMAP_HEADER start */
254/* Definition of member fields of coverage map header.
255 */
256#ifndef COVMAP_HEADER
257#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
258#else
259#define INSTR_PROF_DATA_DEFINED
260#endif
261COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
262 llvm::ConstantInt::get(Int32Ty, NRecords))
263COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
264 llvm::ConstantInt::get(Int32Ty, FilenamesSize))
265COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
266 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
267COVMAP_HEADER(uint32_t, Int32Ty, Version, \
268 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
269#undef COVMAP_HEADER
270/* COVMAP_HEADER end. */
271
272
273#ifdef INSTR_PROF_SECT_ENTRY
274#define INSTR_PROF_DATA_DEFINED
275INSTR_PROF_SECT_ENTRY(IPSK_data, \
276 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
277 INSTR_PROF_DATA_COFF, "__DATA,")
278INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
279 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
280 INSTR_PROF_CNTS_COFF, "__DATA,")
281INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
282 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
283 INSTR_PROF_BITS_COFF, "__DATA,")
284INSTR_PROF_SECT_ENTRY(IPSK_name, \
285 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
286 INSTR_PROF_NAME_COFF, "__DATA,")
287INSTR_PROF_SECT_ENTRY(IPSK_vals, \
288 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
289 INSTR_PROF_VALS_COFF, "__DATA,")
290INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
291 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
292 INSTR_PROF_VNODES_COFF, "__DATA,")
293INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
294 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
295 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
296INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
297 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
298 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
299INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
300 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
301 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
302INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
303 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
304 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
305INSTR_PROF_SECT_ENTRY(IPSK_covname, \
306 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
307 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")
308
309#undef INSTR_PROF_SECT_ENTRY
310#endif
311
312
313#ifdef INSTR_PROF_VALUE_PROF_DATA
314#define INSTR_PROF_DATA_DEFINED
315
316#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
317/*!
318 * This is the header of the data structure that defines the on-disk
319 * layout of the value profile data of a particular kind for one function.
320 */
321typedef struct ValueProfRecord {
322 /* The kind of the value profile record. */
324 /*
325 * The number of value profile sites. It is guaranteed to be non-zero;
326 * otherwise the record for this kind won't be emitted.
327 */
328 uint32_t NumValueSites;
329 /*
330 * The first element of the array that stores the number of profiled
331 * values for each value site. The size of the array is NumValueSites.
332 * Since NumValueSites is greater than zero, there is at least one
333 * element in the array.
334 */
335 uint8_t SiteCountArray[1];
336
337 /*
338 * The fake declaration is for documentation purpose only.
339 * Align the start of next field to be on 8 byte boundaries.
340 uint8_t Padding[X];
341 */
342
343 /* The array of value profile data. The size of the array is the sum
344 * of all elements in SiteCountArray[].
345 InstrProfValueData ValueData[];
346 */
347
348#ifdef __cplusplus
349 /*!
350 * Return the number of value sites.
351 */
352 uint32_t getNumValueSites() const { return NumValueSites; }
353 /*!
354 * Read data from this record and save it to Record.
355 */
356 void deserializeTo(InstrProfRecord &Record,
357 InstrProfSymtab *SymTab);
358 /*
359 * In-place byte swap:
360 * Do byte swap for this instance. \c Old is the original order before
361 * the swap, and \c New is the New byte order.
362 */
364#endif
365} ValueProfRecord;
366
367/*!
368 * Per-function header/control data structure for value profiling
369 * data in indexed format.
370 */
371typedef struct ValueProfData {
372 /*
373 * Total size in bytes including this field. It must be a multiple
374 * of sizeof(uint64_t).
375 */
376 uint32_t TotalSize;
377 /*
378 *The number of value profile kinds that has value profile data.
379 * In this implementation, a value profile kind is considered to
380 * have profile data if the number of value profile sites for the
381 * kind is not zero. More aggressively, the implementation can
382 * choose to check the actual data value: if none of the value sites
383 * has any profiled values, the kind can be skipped.
384 */
385 uint32_t NumValueKinds;
386
387 /*
388 * Following are a sequence of variable length records. The prefix/header
389 * of each record is defined by ValueProfRecord type. The number of
390 * records is NumValueKinds.
391 * ValueProfRecord Record_1;
392 * ValueProfRecord Record_N;
393 */
394
395#if __cplusplus
396 /*!
397 * Return the total size in bytes of the on-disk value profile data
398 * given the data stored in Record.
399 */
400 static uint32_t getSize(const InstrProfRecord &Record);
401 /*!
402 * Return a pointer to \c ValueProfData instance ready to be streamed.
403 */
404 static std::unique_ptr<ValueProfData>
405 serializeFrom(const InstrProfRecord &Record);
406 /*!
407 * Check the integrity of the record.
408 */
409 Error checkIntegrity();
410 /*!
411 * Return a pointer to \c ValueProfileData instance ready to be read.
412 * All data in the instance are properly byte swapped. The input
413 * data is assumed to be in little endian order.
414 */
415 static Expected<std::unique_ptr<ValueProfData>>
416 getValueProfData(const unsigned char *SrcBuffer,
417 const unsigned char *const SrcBufferEnd,
418 llvm::endianness SrcDataEndianness);
419 /*!
420 * Swap byte order from \c Endianness order to host byte order.
421 */
422 void swapBytesToHost(llvm::endianness Endianness);
423 /*!
424 * Swap byte order from host byte order to \c Endianness order.
425 */
426 void swapBytesFromHost(llvm::endianness Endianness);
427 /*!
428 * Return the total size of \c ValueProfileData.
429 */
430 uint32_t getSize() const { return TotalSize; }
431 /*!
432 * Read data from this data and save it to \c Record.
433 */
434 void deserializeTo(InstrProfRecord &Record,
435 InstrProfSymtab *SymTab);
436 void operator delete(void *ptr) { ::operator delete(ptr); }
437#endif
438} ValueProfData;
439
440/*
441 * The closure is designed to abstact away two types of value profile data:
442 * - InstrProfRecord which is the primary data structure used to
443 * represent profile data in host tools (reader, writer, and profile-use)
444 * - value profile runtime data structure suitable to be used by C
445 * runtime library.
446 *
447 * Both sources of data need to serialize to disk/memory-buffer in common
448 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
449 * writer to share the same format and code with indexed profile writer.
450 *
451 * For documentation of the member methods below, refer to corresponding methods
452 * in class InstrProfRecord.
453 */
454typedef struct ValueProfRecordClosure {
455 const void *Record;
456 uint32_t (*GetNumValueKinds)(const void *Record);
457 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
458 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
459 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
460
461 /*
462 * After extracting the value profile data from the value profile record,
463 * this method is used to map the in-memory value to on-disk value. If
464 * the method is null, value will be written out untranslated.
465 */
466 uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
467 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
468 uint32_t S);
469 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
470} ValueProfRecordClosure;
471
472INSTR_PROF_VISIBILITY ValueProfRecord *
473getFirstValueProfRecord(ValueProfData *VPD);
474INSTR_PROF_VISIBILITY ValueProfRecord *
475getValueProfRecordNext(ValueProfRecord *VPR);
476INSTR_PROF_VISIBILITY InstrProfValueData *
477getValueProfRecordValueData(ValueProfRecord *VPR);
478INSTR_PROF_VISIBILITY uint32_t
479getValueProfRecordHeaderSize(uint32_t NumValueSites);
480
481#undef INSTR_PROF_VALUE_PROF_DATA
482#endif /* INSTR_PROF_VALUE_PROF_DATA */
483
484
485#ifdef INSTR_PROF_COMMON_API_IMPL
486#define INSTR_PROF_DATA_DEFINED
487#ifdef __cplusplus
488#define INSTR_PROF_INLINE inline
489#define INSTR_PROF_NULLPTR nullptr
490#else
491#define INSTR_PROF_INLINE
492#define INSTR_PROF_NULLPTR NULL
493#endif
494
495#ifndef offsetof
496#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
497#endif
498
499// clang-format on
500
501/*!
502 * Return the \c ValueProfRecord header size including the
503 * padding bytes.
504 */
505INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
506getValueProfRecordHeaderSize(uint32_t NumValueSites) {
507 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
508 sizeof(uint8_t) * NumValueSites;
509 /* Round the size to multiple of 8 bytes. */
510 Size = (Size + 7) & ~7;
511 return Size;
512}
513
514/*!
515 * Return the total size of the value profile record including the
516 * header and the value data.
517 */
518INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
519getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) {
520 return getValueProfRecordHeaderSize(NumValueSites) +
521 sizeof(InstrProfValueData) * NumValueData;
522}
523
524/*!
525 * Return the pointer to the start of value data array.
526 */
527INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *
528getValueProfRecordValueData(ValueProfRecord *This) {
529 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
530 This->NumValueSites));
531}
532
533/*!
534 * Return the total number of value data for \c This record.
535 */
536INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
537getValueProfRecordNumValueData(ValueProfRecord *This) {
538 uint32_t NumValueData = 0;
539 uint32_t I;
540 for (I = 0; I < This->NumValueSites; I++)
541 NumValueData += This->SiteCountArray[I];
542 return NumValueData;
543}
544
545/*!
546 * Use this method to advance to the next \c This \c ValueProfRecord.
547 */
548INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
549getValueProfRecordNext(ValueProfRecord *This) {
550 uint32_t NumValueData = getValueProfRecordNumValueData(This);
551 return (ValueProfRecord *)((char *)This +
552 getValueProfRecordSize(This->NumValueSites,
553 NumValueData));
554}
555
556/*!
557 * Return the first \c ValueProfRecord instance.
558 */
559INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
560getFirstValueProfRecord(ValueProfData *This) {
561 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
562}
563
564/* Closure based interfaces. */
565
566/*!
567 * Return the total size in bytes of the on-disk value profile data
568 * given the data stored in Record.
569 */
570INSTR_PROF_VISIBILITY uint32_t
571getValueProfDataSize(ValueProfRecordClosure *Closure) {
573 uint32_t TotalSize = sizeof(ValueProfData);
574 const void *Record = Closure->Record;
575
576 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
577 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
578 if (!NumValueSites)
579 continue;
580 TotalSize += getValueProfRecordSize(NumValueSites,
581 Closure->GetNumValueData(Record, Kind));
582 }
583 return TotalSize;
584}
585
586/*!
587 * Extract value profile data of a function for the profile kind \c ValueKind
588 * from the \c Closure and serialize the data into \c This record instance.
589 */
590INSTR_PROF_VISIBILITY void
591serializeValueProfRecordFrom(ValueProfRecord *This,
592 ValueProfRecordClosure *Closure,
593 uint32_t ValueKind, uint32_t NumValueSites) {
594 uint32_t S;
595 const void *Record = Closure->Record;
596 This->Kind = ValueKind;
597 This->NumValueSites = NumValueSites;
598 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
599
600 for (S = 0; S < NumValueSites; S++) {
601 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
602 This->SiteCountArray[S] = ND;
603 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
604 DstVD += ND;
605 }
606}
607
608/*!
609 * Extract value profile data of a function from the \c Closure
610 * and serialize the data into \c DstData if it is not NULL or heap
611 * memory allocated by the \c Closure's allocator method. If \c
612 * DstData is not null, the caller is expected to set the TotalSize
613 * in DstData.
614 */
615INSTR_PROF_VISIBILITY ValueProfData *
616serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
617 ValueProfData *DstData) {
619 uint32_t TotalSize =
620 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
621
622 ValueProfData *VPD =
623 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
624
625 VPD->TotalSize = TotalSize;
626 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
627 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
628 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
629 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
630 if (!NumValueSites)
631 continue;
632 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
633 VR = getValueProfRecordNext(VR);
634 }
635 return VPD;
636}
637
638#undef INSTR_PROF_COMMON_API_IMPL
639#endif /* INSTR_PROF_COMMON_API_IMPL */
640
641/*============================================================================*/
642
643// clang-format off:consider re-enabling clang-format if auto-formatted C macros
644// are readable (e.g., after `issue #82426` is fixed)
645#ifndef INSTR_PROF_DATA_DEFINED
646
647#ifndef INSTR_PROF_DATA_INC
648#define INSTR_PROF_DATA_INC
649
650/* Helper macros. */
651#define INSTR_PROF_SIMPLE_QUOTE(x) #x
652#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
653#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
654#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
655
656/* Magic number to detect file format and endianness.
657 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
658 * so that utilities, like strings, don't grab it as a string. 129 is also
659 * invalid UTF-8, and high enough to be interesting.
660 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
661 * for 32-bit platforms.
662 */
663#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
664 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
665 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
666#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
667 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
668 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
669
670/* Raw profile format version (start from 1). */
671#define INSTR_PROF_RAW_VERSION 9
672/* Indexed profile format version (start from 1). */
673#define INSTR_PROF_INDEX_VERSION 11
674/* Coverage mapping format version (start from 0). */
675#define INSTR_PROF_COVMAP_VERSION 6
676
677/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
678 * version for other variants of profile. We set the 8th most significant bit
679 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
680 * generated profile, and 0 if this is a Clang FE generated profile.
681 * 1 in bit 57 indicates there are context-sensitive records in the profile.
682 * The 59th bit indicates whether to use debug info to correlate profiles.
683 * The 60th bit indicates single byte coverage instrumentation.
684 * The 61st bit indicates function entry instrumentation only.
685 * The 62nd bit indicates whether memory profile information is present.
686 * The 63rd bit indicates if this is a temporal profile.
687 */
688#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
689#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
690#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
691#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
692#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
693#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
694#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
695#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
696#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
697#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
698#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
699#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
700#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
701#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
702
703/* The variable that holds the name of the profile data
704 * specified via command line. */
705#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
706
707/* section name strings common to all targets other
708 than WIN32 */
709#define INSTR_PROF_DATA_COMMON __llvm_prf_data
710#define INSTR_PROF_NAME_COMMON __llvm_prf_names
711#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
712#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
713#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
714#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
715#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
716#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
717#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
718#define INSTR_PROF_COVNAME_COMMON __llvm_covnames
719#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
720/* Windows section names. Because these section names contain dollar characters,
721 * they must be quoted.
722 */
723#define INSTR_PROF_DATA_COFF ".lprfd$M"
724#define INSTR_PROF_NAME_COFF ".lprfn$M"
725#define INSTR_PROF_CNTS_COFF ".lprfc$M"
726#define INSTR_PROF_BITS_COFF ".lprfb$M"
727#define INSTR_PROF_VALS_COFF ".lprfv$M"
728#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
729#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
730#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
731/* Since cov data and cov names sections are not allocated, we don't need to
732 * access them at runtime.
733 */
734#define INSTR_PROF_COVDATA_COFF ".lcovd"
735#define INSTR_PROF_COVNAME_COFF ".lcovn"
736#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
737
738#ifdef _WIN32
739/* Runtime section names and name strings. */
740#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
741#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
742#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
743#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
744/* Array of pointers. Each pointer points to a list
745 * of value nodes associated with one value site.
746 */
747#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
748/* Value profile nodes section. */
749#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
750#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
751#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
752#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
753#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
754#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
755#else
756/* Runtime section names and name strings. */
757#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
758#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
759#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
760#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
761/* Array of pointers. Each pointer points to a list
762 * of value nodes associated with one value site.
763 */
764#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
765/* Value profile nodes section. */
766#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
767#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
768#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
769#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
770#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
771/* Order file instrumentation. */
772#define INSTR_PROF_ORDERFILE_SECT_NAME \
773 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
774#endif
775
776#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
777#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
778 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
779#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
780#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
781 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
782
783/* Macros to define start/stop section symbol for a given
784 * section on Linux. For instance
785 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
786 * expand to __start___llvm_prof_data
787 */
788#define INSTR_PROF_SECT_START(Sect) \
789 INSTR_PROF_CONCAT(__start_,Sect)
790#define INSTR_PROF_SECT_STOP(Sect) \
791 INSTR_PROF_CONCAT(__stop_,Sect)
792
793/* Value Profiling API linkage name. */
794#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
795#define INSTR_PROF_VALUE_PROF_FUNC_STR \
796 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
797#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
798#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
799 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
800
801/* InstrProfile per-function control data alignment. */
802#define INSTR_PROF_DATA_ALIGNMENT 8
803
804/* The data structure that represents a tracked value by the
805 * value profiler.
806 */
807typedef struct InstrProfValueData {
808 /* Profiled value. */
810 /* Number of times the value appears in the training run. */
811 uint64_t Count;
812} InstrProfValueData;
813
814#endif /* INSTR_PROF_DATA_INC */
815
816#ifndef INSTR_ORDER_FILE_INC
817/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
818#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
819#define INSTR_ORDER_FILE_BUFFER_BITS 17
820#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
821#endif /* INSTR_ORDER_FILE_INC */
822#else
823#undef INSTR_PROF_DATA_DEFINED
824#endif
825
826#undef COVMAP_V2_OR_V3
827
828#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
829
830#ifdef __cplusplus
831#define INSTR_PROF_INLINE inline
832#else
833#define INSTR_PROF_INLINE
834#endif
835
836/* The value range buckets (22 buckets) for the memop size value profiling looks
837 * like:
838 *
839 * [0, 0]
840 * [1, 1]
841 * [2, 2]
842 * [3, 3]
843 * [4, 4]
844 * [5, 5]
845 * [6, 6]
846 * [7, 7]
847 * [8, 8]
848 * [9, 15]
849 * [16, 16]
850 * [17, 31]
851 * [32, 32]
852 * [33, 63]
853 * [64, 64]
854 * [65, 127]
855 * [128, 128]
856 * [129, 255]
857 * [256, 256]
858 * [257, 511]
859 * [512, 512]
860 * [513, UINT64_MAX]
861 *
862 * Each range has a 'representative value' which is the lower end value of the
863 * range and used to store in the runtime profile data records and the VP
864 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
865 */
866#define INSTR_PROF_NUM_BUCKETS 22
867
868/*
869 * Clz and Popcount. This code was copied from
870 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
871 * llvm/include/llvm/Support/MathExtras.h.
872 */
873#if defined(_MSC_VER) && !defined(__clang__)
874
875#include <intrin.h>
876INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
877int InstProfClzll(unsigned long long X) {
878 unsigned long LeadZeroIdx = 0;
879#if !defined(_M_ARM64) && !defined(_M_X64)
880 // Scan the high 32 bits.
881 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
882 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
883 // from the MSB.
884 // Scan the low 32 bits.
885 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
886 return (int)(63 - LeadZeroIdx);
887#else
888 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
889#endif
890 return 64;
891}
892INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
893int InstProfPopcountll(unsigned long long X) {
894 // This code originates from https://reviews.llvm.org/rG30626254510f.
895 unsigned long long v = X;
896 v = v - ((v >> 1) & 0x5555555555555555ULL);
897 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
898 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
899 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
900}
901
902#else
903
904INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
905int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
906INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
907int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
908
909#endif /* defined(_MSC_VER) && !defined(__clang__) */
910
911// clang-format on
912
913/* Map an (observed) memop size value to the representative value of its range.
914 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
915INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
916InstrProfGetRangeRepValue(uint64_t Value) {
917 if (Value <= 8)
918 // The first ranges are individually tracked. Use the value as is.
919 return Value;
920 else if (Value >= 513)
921 // The last range is mapped to its lowest value.
922 return 513;
923 else if (InstProfPopcountll(Value) == 1)
924 // If it's a power of two, use it as is.
925 return Value;
926 else
927 // Otherwise, take to the previous power of two + 1.
928 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
929}
930
931/* Return true if the range that an (observed) memop size value belongs to has
932 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
933 * false, 64 -> true, 100 -> false, 513 -> false. */
934INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
935InstrProfIsSingleValRange(uint64_t Value) {
936 if (Value <= 8)
937 // The first ranges are individually tracked.
938 return 1;
939 else if (InstProfPopcountll(Value) == 1)
940 // If it's a power of two, there's only one value.
941 return 1;
942 else
943 // Otherwise, there's more than one value in the range.
944 return 0;
945}
946
947#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
aarch64 promote const
Lower uses of LDS variables from non kernel functions
#define offsetof(TYPE, MEMBER)
RelocType Type
Definition: COFFYAML.cpp:391
#define COVMAP_HEADER(Type, LLVMType, Name, Init)
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init)
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
#define INSTR_PROF_RAW_HEADER(Type, Name, Init)
Definition: InstrProf.h:1205
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)
Definition: InstrProf.h:59
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init)
Definition: InstrProf.h:1196
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
Definition: InstrProf.h:259
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init)
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType)
#define I(x, y, z)
Definition: MD5.cpp:58
IntegerType * Int32Ty
static unsigned getSize(unsigned Kind)
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:647
static Constant * getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy)
getRaw() constructor - Return a constant with array type with an element count and element type match...
Definition: Constants.h:723
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2140
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
ValueKind
Value kinds.
std::enable_if_t< std::is_arithmetic< T >::value, void > swapBytes(T &value)
Definition: DXContainer.h:32
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:622
endianness
Definition: bit.h:70