LLVM  12.0.0git
InstrProfData.inc
Go to the documentation of this file.
1 /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8 /*
9  * This is the master file that defines all the data structure, signature,
10  * constant literals that are shared across profiling runtime library,
11  * compiler (instrumentation), and host tools (reader/writer). The entities
12  * defined in this file affect the profile runtime ABI, the raw profile format,
13  * or both.
14  *
15  * The file has two identical copies. The master copy lives in LLVM and
16  * the other one sits in compiler-rt/lib/profile directory. To make changes
17  * in this file, first modify the master copy and copy it over to compiler-rt.
18  * Testing of any change in this file can start only after the two copies are
19  * synced up.
20  *
21  * The first part of the file includes macros that defines types, names, and
22  * initializers for the member fields of the core data structures. The field
23  * declarations for one structure is enabled by defining the field activation
24  * macro associated with that structure. Only one field activation record
25  * can be defined at one time and the rest definitions will be filtered out by
26  * the preprocessor.
27  *
28  * Examples of how the template is used to instantiate structure definition:
29  * 1. To declare a structure:
30  *
31  * struct ProfData {
32  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33  * Type Name;
34  * #include "llvm/ProfileData/InstrProfData.inc"
35  * };
36  *
37  * 2. To construct LLVM type arrays for the struct type:
38  *
39  * Type *DataTypes[] = {
40  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41  * LLVMType,
42  * #include "llvm/ProfileData/InstrProfData.inc"
43  * };
44  *
45  * 4. To construct constant array for the initializers:
46  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47  * Initializer,
48  * Constant *ConstantVals[] = {
49  * #include "llvm/ProfileData/InstrProfData.inc"
50  * };
51  *
52  *
53  * The second part of the file includes definitions all other entities that
54  * are related to runtime ABI and format. When no field activation macro is
55  * defined, this file can be included to introduce the definitions.
56  *
57 \*===----------------------------------------------------------------------===*/
58 
59 /* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60  * the compiler runtime. */
61 #ifndef INSTR_PROF_VISIBILITY
62 #define INSTR_PROF_VISIBILITY
63 #endif
64 
65 /* INSTR_PROF_DATA start. */
66 /* Definition of member fields of the per-function control structure. */
67 #ifndef INSTR_PROF_DATA
68 #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69 #else
70 #define INSTR_PROF_DATA_DEFINED
71 #endif
72 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
73  ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
75 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76  ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77  Inc->getHash()->getZExtValue()))
78 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
79  ConstantExpr::getBitCast(CounterPtr, \
80  llvm::Type::getInt64PtrTy(Ctx)))
81 /* This is used to map function pointers for the indirect call targets to
82  * function name hashes during the conversion from raw to merged profile
83  * data.
84  */
85 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
86  FunctionAddr)
87 INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
88  ValuesPtrExpr)
89 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
90  ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
91 INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
92  ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
93 #undef INSTR_PROF_DATA
94 /* INSTR_PROF_DATA end. */
95 
96 
97 /* This is an internal data structure used by value profiler. It
98  * is defined here to allow serialization code sharing by LLVM
99  * to be used in unit test.
100  *
101  * typedef struct ValueProfNode {
102  * // InstrProfValueData VData;
103  * uint64_t Value;
104  * uint64_t Count;
105  * struct ValueProfNode *Next;
106  * } ValueProfNode;
107  */
108 /* INSTR_PROF_VALUE_NODE start. */
109 #ifndef INSTR_PROF_VALUE_NODE
110 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
111 #else
112 #define INSTR_PROF_DATA_DEFINED
113 #endif
115  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
116 INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
117  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
118 INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
119  ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
120 #undef INSTR_PROF_VALUE_NODE
121 /* INSTR_PROF_VALUE_NODE end. */
122 
123 /* INSTR_PROF_RAW_HEADER start */
124 /* Definition of member fields of the raw profile header data structure. */
125 #ifndef INSTR_PROF_RAW_HEADER
126 #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
127 #else
128 #define INSTR_PROF_DATA_DEFINED
129 #endif
130 INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
131 INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
132 INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
133 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
134 INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
135 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
136 INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
137 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
138 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
139 INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
140 #undef INSTR_PROF_RAW_HEADER
141 /* INSTR_PROF_RAW_HEADER end */
142 
143 /* VALUE_PROF_FUNC_PARAM start */
144 /* Definition of parameter types of the runtime API used to do value profiling
145  * for a given value site.
146  */
147 #ifndef VALUE_PROF_FUNC_PARAM
148 #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
149 #define INSTR_PROF_COMMA
150 #else
151 #define INSTR_PROF_DATA_DEFINED
152 #define INSTR_PROF_COMMA ,
153 #endif
154 VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
155  INSTR_PROF_COMMA
156 VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
157 #ifndef VALUE_RANGE_PROF
158 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
159 #else /* VALUE_RANGE_PROF */
160 /* FIXME: This is to be removed after switching to the new memop value
161  * profiling. */
162 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
163  INSTR_PROF_COMMA
164 VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
165  INSTR_PROF_COMMA
166 VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \
167  INSTR_PROF_COMMA
168 VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx))
169 #endif /*VALUE_RANGE_PROF */
170 #undef VALUE_PROF_FUNC_PARAM
171 #undef INSTR_PROF_COMMA
172 /* VALUE_PROF_FUNC_PARAM end */
173 
174 /* VALUE_PROF_KIND start */
175 #ifndef VALUE_PROF_KIND
176 #define VALUE_PROF_KIND(Enumerator, Value, Descr)
177 #else
178 #define INSTR_PROF_DATA_DEFINED
179 #endif
180 /* For indirect function call value profiling, the addresses of the target
181  * functions are profiled by the instrumented code. The target addresses are
182  * written in the raw profile data and converted to target function name's MD5
183  * hash by the profile reader during deserialization. Typically, this happens
184  * when the raw profile data is read during profile merging.
185  *
186  * For this remapping the ProfData is used. ProfData contains both the function
187  * name hash and the function address.
188  */
189 VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
190 /* For memory intrinsic functions size profiling. */
191 VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
192 /* These two kinds must be the last to be
193  * declared. This is to make sure the string
194  * array created with the template can be
195  * indexed with the kind value.
196  */
197 VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
198 VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
199 
200 #undef VALUE_PROF_KIND
201 /* VALUE_PROF_KIND end */
202 
203 #undef COVMAP_V2_OR_V3
204 #ifdef COVMAP_V2
205 #define COVMAP_V2_OR_V3
206 #endif
207 #ifdef COVMAP_V3
208 #define COVMAP_V2_OR_V3
209 #endif
210 
211 /* COVMAP_FUNC_RECORD start */
212 /* Definition of member fields of the function record structure in coverage
213  * map.
214  */
215 #ifndef COVMAP_FUNC_RECORD
216 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
217 #else
218 #define INSTR_PROF_DATA_DEFINED
219 #endif
220 #ifdef COVMAP_V1
221 COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
222  NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
224 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
226  NameValue.size()))
227 #endif
228 #ifdef COVMAP_V2_OR_V3
229 COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
231  llvm::Type::getInt64Ty(Ctx), NameHash))
232 #endif
233 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
235  llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
236 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
237  llvm::ConstantInt::get( \
238  llvm::Type::getInt64Ty(Ctx), FuncHash))
239 #ifdef COVMAP_V3
240 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
242  llvm::Type::getInt64Ty(Ctx), FilenamesRef))
243 COVMAP_FUNC_RECORD(const char, \
245  CoverageMapping.size()), \
246  CoverageMapping,
248  CoverageMapping, CoverageMapping.size(), \
250 #endif
251 #undef COVMAP_FUNC_RECORD
252 /* COVMAP_FUNC_RECORD end. */
253 
254 /* COVMAP_HEADER start */
255 /* Definition of member fields of coverage map header.
256  */
257 #ifndef COVMAP_HEADER
258 #define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
259 #else
260 #define INSTR_PROF_DATA_DEFINED
261 #endif
262 COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
263  llvm::ConstantInt::get(Int32Ty, NRecords))
264 COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
265  llvm::ConstantInt::get(Int32Ty, FilenamesSize))
266 COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
267  llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
268 COVMAP_HEADER(uint32_t, Int32Ty, Version, \
270 #undef COVMAP_HEADER
271 /* COVMAP_HEADER end. */
272 
273 
274 #ifdef INSTR_PROF_SECT_ENTRY
275 #define INSTR_PROF_DATA_DEFINED
276 INSTR_PROF_SECT_ENTRY(IPSK_data, \
277  INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
278  INSTR_PROF_DATA_COFF, "__DATA,")
279 INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
280  INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
281  INSTR_PROF_CNTS_COFF, "__DATA,")
282 INSTR_PROF_SECT_ENTRY(IPSK_name, \
283  INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
284  INSTR_PROF_NAME_COFF, "__DATA,")
285 INSTR_PROF_SECT_ENTRY(IPSK_vals, \
286  INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
287  INSTR_PROF_VALS_COFF, "__DATA,")
288 INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
289  INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
290  INSTR_PROF_VNODES_COFF, "__DATA,")
291 INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
292  INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
293  INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
294 INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
295  INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
296  INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
297 INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
298  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
299  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
300 
301 #undef INSTR_PROF_SECT_ENTRY
302 #endif
303 
304 
305 #ifdef INSTR_PROF_VALUE_PROF_DATA
306 #define INSTR_PROF_DATA_DEFINED
307 
308 #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
309 /*!
310  * This is the header of the data structure that defines the on-disk
311  * layout of the value profile data of a particular kind for one function.
312  */
313 typedef struct ValueProfRecord {
314  /* The kind of the value profile record. */
315  uint32_t Kind;
316  /*
317  * The number of value profile sites. It is guaranteed to be non-zero;
318  * otherwise the record for this kind won't be emitted.
319  */
320  uint32_t NumValueSites;
321  /*
322  * The first element of the array that stores the number of profiled
323  * values for each value site. The size of the array is NumValueSites.
324  * Since NumValueSites is greater than zero, there is at least one
325  * element in the array.
326  */
327  uint8_t SiteCountArray[1];
328 
329  /*
330  * The fake declaration is for documentation purpose only.
331  * Align the start of next field to be on 8 byte boundaries.
332  uint8_t Padding[X];
333  */
334 
335  /* The array of value profile data. The size of the array is the sum
336  * of all elements in SiteCountArray[].
337  InstrProfValueData ValueData[];
338  */
339 
340 #ifdef __cplusplus
341  /*!
342  * Return the number of value sites.
343  */
344  uint32_t getNumValueSites() const { return NumValueSites; }
345  /*!
346  * Read data from this record and save it to Record.
347  */
348  void deserializeTo(InstrProfRecord &Record,
349  InstrProfSymtab *SymTab);
350  /*
351  * In-place byte swap:
352  * Do byte swap for this instance. \c Old is the original order before
353  * the swap, and \c New is the New byte order.
354  */
355  void swapBytes(support::endianness Old, support::endianness New);
356 #endif
357 } ValueProfRecord;
358 
359 /*!
360  * Per-function header/control data structure for value profiling
361  * data in indexed format.
362  */
363 typedef struct ValueProfData {
364  /*
365  * Total size in bytes including this field. It must be a multiple
366  * of sizeof(uint64_t).
367  */
368  uint32_t TotalSize;
369  /*
370  *The number of value profile kinds that has value profile data.
371  * In this implementation, a value profile kind is considered to
372  * have profile data if the number of value profile sites for the
373  * kind is not zero. More aggressively, the implementation can
374  * choose to check the actual data value: if none of the value sites
375  * has any profiled values, the kind can be skipped.
376  */
377  uint32_t NumValueKinds;
378 
379  /*
380  * Following are a sequence of variable length records. The prefix/header
381  * of each record is defined by ValueProfRecord type. The number of
382  * records is NumValueKinds.
383  * ValueProfRecord Record_1;
384  * ValueProfRecord Record_N;
385  */
386 
387 #if __cplusplus
388  /*!
389  * Return the total size in bytes of the on-disk value profile data
390  * given the data stored in Record.
391  */
392  static uint32_t getSize(const InstrProfRecord &Record);
393  /*!
394  * Return a pointer to \c ValueProfData instance ready to be streamed.
395  */
396  static std::unique_ptr<ValueProfData>
397  serializeFrom(const InstrProfRecord &Record);
398  /*!
399  * Check the integrity of the record.
400  */
401  Error checkIntegrity();
402  /*!
403  * Return a pointer to \c ValueProfileData instance ready to be read.
404  * All data in the instance are properly byte swapped. The input
405  * data is assumed to be in little endian order.
406  */
407  static Expected<std::unique_ptr<ValueProfData>>
408  getValueProfData(const unsigned char *SrcBuffer,
409  const unsigned char *const SrcBufferEnd,
410  support::endianness SrcDataEndianness);
411  /*!
412  * Swap byte order from \c Endianness order to host byte order.
413  */
414  void swapBytesToHost(support::endianness Endianness);
415  /*!
416  * Swap byte order from host byte order to \c Endianness order.
417  */
418  void swapBytesFromHost(support::endianness Endianness);
419  /*!
420  * Return the total size of \c ValueProfileData.
421  */
422  uint32_t getSize() const { return TotalSize; }
423  /*!
424  * Read data from this data and save it to \c Record.
425  */
426  void deserializeTo(InstrProfRecord &Record,
427  InstrProfSymtab *SymTab);
428  void operator delete(void *ptr) { ::operator delete(ptr); }
429 #endif
430 } ValueProfData;
431 
432 /*
433  * The closure is designed to abstact away two types of value profile data:
434  * - InstrProfRecord which is the primary data structure used to
435  * represent profile data in host tools (reader, writer, and profile-use)
436  * - value profile runtime data structure suitable to be used by C
437  * runtime library.
438  *
439  * Both sources of data need to serialize to disk/memory-buffer in common
440  * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
441  * writer to share the same format and code with indexed profile writer.
442  *
443  * For documentation of the member methods below, refer to corresponding methods
444  * in class InstrProfRecord.
445  */
446 typedef struct ValueProfRecordClosure {
447  const void *Record;
448  uint32_t (*GetNumValueKinds)(const void *Record);
449  uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
450  uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
451  uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
452 
453  /*
454  * After extracting the value profile data from the value profile record,
455  * this method is used to map the in-memory value to on-disk value. If
456  * the method is null, value will be written out untranslated.
457  */
458  uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
459  void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
460  uint32_t S);
461  ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
462 } ValueProfRecordClosure;
463 
464 INSTR_PROF_VISIBILITY ValueProfRecord *
465 getFirstValueProfRecord(ValueProfData *VPD);
466 INSTR_PROF_VISIBILITY ValueProfRecord *
467 getValueProfRecordNext(ValueProfRecord *VPR);
468 INSTR_PROF_VISIBILITY InstrProfValueData *
469 getValueProfRecordValueData(ValueProfRecord *VPR);
470 INSTR_PROF_VISIBILITY uint32_t
471 getValueProfRecordHeaderSize(uint32_t NumValueSites);
472 
473 #undef INSTR_PROF_VALUE_PROF_DATA
474 #endif /* INSTR_PROF_VALUE_PROF_DATA */
475 
476 
477 #ifdef INSTR_PROF_COMMON_API_IMPL
478 #define INSTR_PROF_DATA_DEFINED
479 #ifdef __cplusplus
480 #define INSTR_PROF_INLINE inline
481 #define INSTR_PROF_NULLPTR nullptr
482 #else
483 #define INSTR_PROF_INLINE
484 #define INSTR_PROF_NULLPTR NULL
485 #endif
486 
487 #ifndef offsetof
488 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
489 #endif
490 
491 /*!
492  * Return the \c ValueProfRecord header size including the
493  * padding bytes.
494  */
495 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
496 uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
497  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
498  sizeof(uint8_t) * NumValueSites;
499  /* Round the size to multiple of 8 bytes. */
500  Size = (Size + 7) & ~7;
501  return Size;
502 }
503 
504 /*!
505  * Return the total size of the value profile record including the
506  * header and the value data.
507  */
508 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
509 uint32_t getValueProfRecordSize(uint32_t NumValueSites,
510  uint32_t NumValueData) {
511  return getValueProfRecordHeaderSize(NumValueSites) +
512  sizeof(InstrProfValueData) * NumValueData;
513 }
514 
515 /*!
516  * Return the pointer to the start of value data array.
517  */
518 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
519 InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
520  return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
521  This->NumValueSites));
522 }
523 
524 /*!
525  * Return the total number of value data for \c This record.
526  */
527 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
528 uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
529  uint32_t NumValueData = 0;
530  uint32_t I;
531  for (I = 0; I < This->NumValueSites; I++)
532  NumValueData += This->SiteCountArray[I];
533  return NumValueData;
534 }
535 
536 /*!
537  * Use this method to advance to the next \c This \c ValueProfRecord.
538  */
539 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
540 ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
541  uint32_t NumValueData = getValueProfRecordNumValueData(This);
542  return (ValueProfRecord *)((char *)This +
543  getValueProfRecordSize(This->NumValueSites,
544  NumValueData));
545 }
546 
547 /*!
548  * Return the first \c ValueProfRecord instance.
549  */
550 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
551 ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
552  return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
553 }
554 
555 /* Closure based interfaces. */
556 
557 /*!
558  * Return the total size in bytes of the on-disk value profile data
559  * given the data stored in Record.
560  */
561 INSTR_PROF_VISIBILITY uint32_t
562 getValueProfDataSize(ValueProfRecordClosure *Closure) {
563  uint32_t Kind;
564  uint32_t TotalSize = sizeof(ValueProfData);
565  const void *Record = Closure->Record;
566 
567  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
568  uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
569  if (!NumValueSites)
570  continue;
571  TotalSize += getValueProfRecordSize(NumValueSites,
572  Closure->GetNumValueData(Record, Kind));
573  }
574  return TotalSize;
575 }
576 
577 /*!
578  * Extract value profile data of a function for the profile kind \c ValueKind
579  * from the \c Closure and serialize the data into \c This record instance.
580  */
581 INSTR_PROF_VISIBILITY void
582 serializeValueProfRecordFrom(ValueProfRecord *This,
583  ValueProfRecordClosure *Closure,
584  uint32_t ValueKind, uint32_t NumValueSites) {
585  uint32_t S;
586  const void *Record = Closure->Record;
587  This->Kind = ValueKind;
588  This->NumValueSites = NumValueSites;
589  InstrProfValueData *DstVD = getValueProfRecordValueData(This);
590 
591  for (S = 0; S < NumValueSites; S++) {
592  uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
593  This->SiteCountArray[S] = ND;
594  Closure->GetValueForSite(Record, DstVD, ValueKind, S);
595  DstVD += ND;
596  }
597 }
598 
599 /*!
600  * Extract value profile data of a function from the \c Closure
601  * and serialize the data into \c DstData if it is not NULL or heap
602  * memory allocated by the \c Closure's allocator method. If \c
603  * DstData is not null, the caller is expected to set the TotalSize
604  * in DstData.
605  */
606 INSTR_PROF_VISIBILITY ValueProfData *
607 serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
608  ValueProfData *DstData) {
609  uint32_t Kind;
610  uint32_t TotalSize =
611  DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
612 
613  ValueProfData *VPD =
614  DstData ? DstData : Closure->AllocValueProfData(TotalSize);
615 
616  VPD->TotalSize = TotalSize;
617  VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
618  ValueProfRecord *VR = getFirstValueProfRecord(VPD);
619  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
620  uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
621  if (!NumValueSites)
622  continue;
623  serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
624  VR = getValueProfRecordNext(VR);
625  }
626  return VPD;
627 }
628 
629 #undef INSTR_PROF_COMMON_API_IMPL
630 #endif /* INSTR_PROF_COMMON_API_IMPL */
631 
632 /*============================================================================*/
633 
634 #ifndef INSTR_PROF_DATA_DEFINED
635 
636 #ifndef INSTR_PROF_DATA_INC
637 #define INSTR_PROF_DATA_INC
638 
639 /* Helper macros. */
640 #define INSTR_PROF_SIMPLE_QUOTE(x) #x
641 #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
642 #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
643 #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
644 
645 /* Magic number to detect file format and endianness.
646  * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
647  * so that utilities, like strings, don't grab it as a string. 129 is also
648  * invalid UTF-8, and high enough to be interesting.
649  * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
650  * for 32-bit platforms.
651  */
652 #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
653  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
654  (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
655 #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
656  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
657  (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
658 
659 /* Raw profile format version (start from 1). */
660 #define INSTR_PROF_RAW_VERSION 5
661 /* Indexed profile format version (start from 1). */
662 #define INSTR_PROF_INDEX_VERSION 6
663 /* Coverage mapping format version (start from 0). */
664 #define INSTR_PROF_COVMAP_VERSION 3
665 
666 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
667  * version for other variants of profile. We set the lowest bit of the upper 8
668  * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
669  * generated profile, and 0 if this is a Clang FE generated profile.
670  * 1 in bit 57 indicates there are context-sensitive records in the profile.
671  */
672 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
673 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
674 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
675 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
676 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
677 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
678 
679 /* The variable that holds the name of the profile data
680  * specified via command line. */
681 #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
682 
683 /* section name strings common to all targets other
684  than WIN32 */
685 #define INSTR_PROF_DATA_COMMON __llvm_prf_data
686 #define INSTR_PROF_NAME_COMMON __llvm_prf_names
687 #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
688 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
689 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
690 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
691 #define INSTR_PROF_COVFUN_COMMON __llvm_covfun
692 #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
693 /* Windows section names. Because these section names contain dollar characters,
694  * they must be quoted.
695  */
696 #define INSTR_PROF_DATA_COFF ".lprfd$M"
697 #define INSTR_PROF_NAME_COFF ".lprfn$M"
698 #define INSTR_PROF_CNTS_COFF ".lprfc$M"
699 #define INSTR_PROF_VALS_COFF ".lprfv$M"
700 #define INSTR_PROF_VNODES_COFF ".lprfnd$M"
701 #define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
702 #define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
703 #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
704 
705 #ifdef _WIN32
706 /* Runtime section names and name strings. */
707 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
708 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
709 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
710 /* Array of pointers. Each pointer points to a list
711  * of value nodes associated with one value site.
712  */
713 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
714 /* Value profile nodes section. */
715 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
716 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
717 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
718 #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
719 #else
720 /* Runtime section names and name strings. */
721 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
722 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
723 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
724 /* Array of pointers. Each pointer points to a list
725  * of value nodes associated with one value site.
726  */
727 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
728 /* Value profile nodes section. */
729 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
730 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
731 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
732 /* Order file instrumentation. */
733 #define INSTR_PROF_ORDERFILE_SECT_NAME \
734  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
735 #endif
736 
737 #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
738 #define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
739  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
740 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
741 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
742  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
743 
744 /* Macros to define start/stop section symbol for a given
745  * section on Linux. For instance
746  * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
747  * expand to __start___llvm_prof_data
748  */
749 #define INSTR_PROF_SECT_START(Sect) \
750  INSTR_PROF_CONCAT(__start_,Sect)
751 #define INSTR_PROF_SECT_STOP(Sect) \
752  INSTR_PROF_CONCAT(__stop_,Sect)
753 
754 /* Value Profiling API linkage name. */
755 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
756 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
757  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
758 /* FIXME: This is to be removed after switching to the new memop value
759  * profiling. */
760 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
761 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
762  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
763 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
764 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
765  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
766 
767 /* InstrProfile per-function control data alignment. */
768 #define INSTR_PROF_DATA_ALIGNMENT 8
769 
770 /* The data structure that represents a tracked value by the
771  * value profiler.
772  */
773 typedef struct InstrProfValueData {
774  /* Profiled value. */
775  uint64_t Value;
776  /* Number of times the value appears in the training run. */
777  uint64_t Count;
778 } InstrProfValueData;
779 
780 #endif /* INSTR_PROF_DATA_INC */
781 
782 #ifndef INSTR_ORDER_FILE_INC
783 /* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
784 #define INSTR_ORDER_FILE_BUFFER_SIZE 131072
785 #define INSTR_ORDER_FILE_BUFFER_BITS 17
786 #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
787 #endif /* INSTR_ORDER_FILE_INC */
788 #else
789 #undef INSTR_PROF_DATA_DEFINED
790 #endif
791 
792 #undef COVMAP_V2_OR_V3
793 
794 #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
795 
796 #ifdef __cplusplus
797 #define INSTR_PROF_INLINE inline
798 #else
799 #define INSTR_PROF_INLINE
800 #endif
801 
802 /* The value range buckets (22 buckets) for the memop size value profiling looks
803  * like:
804  *
805  * [0, 0]
806  * [1, 1]
807  * [2, 2]
808  * [3, 3]
809  * [4, 4]
810  * [5, 5]
811  * [6, 6]
812  * [7, 7]
813  * [8, 8]
814  * [9, 15]
815  * [16, 16]
816  * [17, 31]
817  * [32, 32]
818  * [33, 63]
819  * [64, 64]
820  * [65, 127]
821  * [128, 128]
822  * [129, 255]
823  * [256, 256]
824  * [257, 511]
825  * [512, 512]
826  * [513, UINT64_MAX]
827  *
828  * Each range has a 'representative value' which is the lower end value of the
829  * range and used to store in the runtime profile data records and the VP
830  * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
831  */
832 
833 /*
834  * Clz and Popcount. This code was copied from
835  * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
836  * llvm/include/llvm/Support/MathExtras.h.
837  */
838 #if defined(_MSC_VER) && !defined(__clang__)
839 
840 #include <intrin.h>
841 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
842 int InstProfClzll(unsigned long long X) {
843  unsigned long LeadZeroIdx = 0;
844 #if !defined(_M_ARM64) && !defined(_M_X64)
845  // Scan the high 32 bits.
846  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
847  return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
848  // from the MSB.
849  // Scan the low 32 bits.
850  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
851  return (int)(63 - LeadZeroIdx);
852 #else
853  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
854 #endif
855  return 64;
856 }
857 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
858 int InstProfPopcountll(unsigned long long X) {
859  // This code originates from https://reviews.llvm.org/rG30626254510f.
860  unsigned long long v = X;
861  v = v - ((v >> 1) & 0x5555555555555555ULL);
862  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
863  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
864  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
865 }
866 
867 #else
868 
869 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
870 int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
871 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
872 int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
873 
874 #endif /* defined(_MSC_VER) && !defined(__clang__) */
875 
876 /* Map an (observed) memop size value to the representative value of its range.
877  * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
878 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
879 InstrProfGetRangeRepValue(uint64_t Value) {
880  if (Value <= 8)
881  // The first ranges are individually tracked. Use the value as is.
882  return Value;
883  else if (Value >= 513)
884  // The last range is mapped to its lowest value.
885  return 513;
886  else if (InstProfPopcountll(Value) == 1)
887  // If it's a power of two, use it as is.
888  return Value;
889  else
890  // Otherwise, take to the previous power of two + 1.
891  return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
892 }
893 
894 /* Return true if the range that an (observed) memop size value belongs to has
895  * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
896  * false, 64 -> true, 100 -> false, 513 -> false. */
897 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
898 InstrProfIsSingleValRange(uint64_t Value) {
899  if (Value <= 8)
900  // The first ranges are individually tracked.
901  return 1;
902  else if (InstProfPopcountll(Value) == 1)
903  // If it's a power of two, there's only one value.
904  return 1;
905  else
906  // Otherwise, there's more than one value in the range.
907  return 0;
908 }
909 
910 #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
Type
MessagePack types as defined in the standard, with the exception of Integer being divided into a sign...
Definition: MsgPackReader.h:49
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This class represents lattice values for constants.
Definition: AllocatorList.h:23
constexpr support::endianness Endianness
The endianness of all multi-byte encoded values in MessagePack.
Definition: MsgPack.h:24
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType)
static Constant * getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy)
get() constructor - Return a constant with array type with an element count and element type matching...
Definition: Constants.h:725
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:187
llvm::support::endianness endianness
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init)
Definition: InstrProf.h:1128
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2025
ValueKind
Value kinds.
#define offsetof(TYPE, MEMBER)
amdgpu propagate attributes Late propagate attributes from kernels to functions
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
Definition: InstrProf.h:246
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1000
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:234
static const char *const Magic
Definition: Archive.cpp:41
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init)
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:786
auto size(R &&Range, std::enable_if_t< std::is_same< typename std::iterator_traits< decltype(Range.begin())>::iterator_category, std::random_access_iterator_tag >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1473
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)
Definition: InstrProf.h:58
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init)
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:186
#define I(x, y, z)
Definition: MD5.cpp:59
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:571
uint32_t Size
Definition: Profile.cpp:46
#define INSTR_PROF_RAW_HEADER(Type, Name, Init)
Definition: InstrProf.h:1137
aarch64 promote const
#define COVMAP_HEADER(Type, LLVMType, Name, Init)
const uint64_t Version
Definition: InstrProf.h:996
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:184
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:423
IntegerType * Int32Ty