LLVM  14.0.0git
InstrProfData.inc
Go to the documentation of this file.
1 /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8 /*
9  * This is the main file that defines all the data structure, signature,
10  * constant literals that are shared across profiling runtime library,
11  * compiler (instrumentation), and host tools (reader/writer). The entities
12  * defined in this file affect the profile runtime ABI, the raw profile format,
13  * or both.
14  *
15  * The file has two identical copies. The primary copy lives in LLVM and
16  * the other one sits in compiler-rt/lib/profile directory. To make changes
17  * in this file, first modify the primary copy and copy it over to compiler-rt.
18  * Testing of any change in this file can start only after the two copies are
19  * synced up.
20  *
21  * The first part of the file includes macros that defines types, names, and
22  * initializers for the member fields of the core data structures. The field
23  * declarations for one structure is enabled by defining the field activation
24  * macro associated with that structure. Only one field activation record
25  * can be defined at one time and the rest definitions will be filtered out by
26  * the preprocessor.
27  *
28  * Examples of how the template is used to instantiate structure definition:
29  * 1. To declare a structure:
30  *
31  * struct ProfData {
32  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33  * Type Name;
34  * #include "llvm/ProfileData/InstrProfData.inc"
35  * };
36  *
37  * 2. To construct LLVM type arrays for the struct type:
38  *
39  * Type *DataTypes[] = {
40  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41  * LLVMType,
42  * #include "llvm/ProfileData/InstrProfData.inc"
43  * };
44  *
45  * 4. To construct constant array for the initializers:
46  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47  * Initializer,
48  * Constant *ConstantVals[] = {
49  * #include "llvm/ProfileData/InstrProfData.inc"
50  * };
51  *
52  *
53  * The second part of the file includes definitions all other entities that
54  * are related to runtime ABI and format. When no field activation macro is
55  * defined, this file can be included to introduce the definitions.
56  *
57 \*===----------------------------------------------------------------------===*/
58 
59 /* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60  * the compiler runtime. */
61 #ifndef INSTR_PROF_VISIBILITY
62 #define INSTR_PROF_VISIBILITY
63 #endif
64 
65 /* INSTR_PROF_DATA start. */
66 /* Definition of member fields of the per-function control structure. */
67 #ifndef INSTR_PROF_DATA
68 #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69 #else
70 #define INSTR_PROF_DATA_DEFINED
71 #endif
72 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
75 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76  ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77  Inc->getHash()->getZExtValue()))
78 INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
79 /* This is used to map function pointers for the indirect call targets to
80  * function name hashes during the conversion from raw to merged profile
81  * data.
82  */
83 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
84  FunctionAddr)
85 INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
86  ValuesPtrExpr)
87 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
88  ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
89 INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
90  ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
91 #undef INSTR_PROF_DATA
92 /* INSTR_PROF_DATA end. */
93 
94 
95 /* This is an internal data structure used by value profiler. It
96  * is defined here to allow serialization code sharing by LLVM
97  * to be used in unit test.
98  *
99  * typedef struct ValueProfNode {
100  * // InstrProfValueData VData;
101  * uint64_t Value;
102  * uint64_t Count;
103  * struct ValueProfNode *Next;
104  * } ValueProfNode;
105  */
106 /* INSTR_PROF_VALUE_NODE start. */
107 #ifndef INSTR_PROF_VALUE_NODE
108 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
109 #else
110 #define INSTR_PROF_DATA_DEFINED
111 #endif
113  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
114 INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
115  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
116 INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
117  ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
118 #undef INSTR_PROF_VALUE_NODE
119 /* INSTR_PROF_VALUE_NODE end. */
120 
121 /* INSTR_PROF_RAW_HEADER start */
122 /* Definition of member fields of the raw profile header data structure. */
123 #ifndef INSTR_PROF_RAW_HEADER
124 #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
125 #else
126 #define INSTR_PROF_DATA_DEFINED
127 #endif
128 INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
129 INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
130 INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
131 /* FIXME: A more accurate name is NumData */
132 INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
133 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
134 /* FIXME: A more accurate name is NumCounters */
135 INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
136 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
137 INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
138 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
139  (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
140 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
141 INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
142 #undef INSTR_PROF_RAW_HEADER
143 /* INSTR_PROF_RAW_HEADER end */
144 
145 /* VALUE_PROF_FUNC_PARAM start */
146 /* Definition of parameter types of the runtime API used to do value profiling
147  * for a given value site.
148  */
149 #ifndef VALUE_PROF_FUNC_PARAM
150 #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
151 #define INSTR_PROF_COMMA
152 #else
153 #define INSTR_PROF_DATA_DEFINED
154 #define INSTR_PROF_COMMA ,
155 #endif
156 VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
157  INSTR_PROF_COMMA
158 VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
159 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
160 #undef VALUE_PROF_FUNC_PARAM
161 #undef INSTR_PROF_COMMA
162 /* VALUE_PROF_FUNC_PARAM end */
163 
164 /* VALUE_PROF_KIND start */
165 #ifndef VALUE_PROF_KIND
166 #define VALUE_PROF_KIND(Enumerator, Value, Descr)
167 #else
168 #define INSTR_PROF_DATA_DEFINED
169 #endif
170 /* For indirect function call value profiling, the addresses of the target
171  * functions are profiled by the instrumented code. The target addresses are
172  * written in the raw profile data and converted to target function name's MD5
173  * hash by the profile reader during deserialization. Typically, this happens
174  * when the raw profile data is read during profile merging.
175  *
176  * For this remapping the ProfData is used. ProfData contains both the function
177  * name hash and the function address.
178  */
179 VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
180 /* For memory intrinsic functions size profiling. */
181 VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
182 /* These two kinds must be the last to be
183  * declared. This is to make sure the string
184  * array created with the template can be
185  * indexed with the kind value.
186  */
187 VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
188 VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
189 
190 #undef VALUE_PROF_KIND
191 /* VALUE_PROF_KIND end */
192 
193 #undef COVMAP_V2_OR_V3
194 #ifdef COVMAP_V2
195 #define COVMAP_V2_OR_V3
196 #endif
197 #ifdef COVMAP_V3
198 #define COVMAP_V2_OR_V3
199 #endif
200 
201 /* COVMAP_FUNC_RECORD start */
202 /* Definition of member fields of the function record structure in coverage
203  * map.
204  */
205 #ifndef COVMAP_FUNC_RECORD
206 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
207 #else
208 #define INSTR_PROF_DATA_DEFINED
209 #endif
210 #ifdef COVMAP_V1
211 COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
212  NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
216  NameValue.size()))
217 #endif
218 #ifdef COVMAP_V2_OR_V3
219 COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
221  llvm::Type::getInt64Ty(Ctx), NameHash))
222 #endif
223 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
225  llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
226 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
227  llvm::ConstantInt::get( \
228  llvm::Type::getInt64Ty(Ctx), FuncHash))
229 #ifdef COVMAP_V3
230 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
232  llvm::Type::getInt64Ty(Ctx), FilenamesRef))
233 COVMAP_FUNC_RECORD(const char, \
235  CoverageMapping.size()), \
236  CoverageMapping,
238  CoverageMapping, CoverageMapping.size(), \
239  llvm::Type::getInt8Ty(Ctx)))
240 #endif
241 #undef COVMAP_FUNC_RECORD
242 /* COVMAP_FUNC_RECORD end. */
243 
244 /* COVMAP_HEADER start */
245 /* Definition of member fields of coverage map header.
246  */
247 #ifndef COVMAP_HEADER
248 #define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
249 #else
250 #define INSTR_PROF_DATA_DEFINED
251 #endif
252 COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
253  llvm::ConstantInt::get(Int32Ty, NRecords))
254 COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
255  llvm::ConstantInt::get(Int32Ty, FilenamesSize))
256 COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
257  llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
260 #undef COVMAP_HEADER
261 /* COVMAP_HEADER end. */
262 
263 
264 #ifdef INSTR_PROF_SECT_ENTRY
265 #define INSTR_PROF_DATA_DEFINED
266 INSTR_PROF_SECT_ENTRY(IPSK_data, \
267  INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
268  INSTR_PROF_DATA_COFF, "__DATA,")
269 INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
270  INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
271  INSTR_PROF_CNTS_COFF, "__DATA,")
272 INSTR_PROF_SECT_ENTRY(IPSK_name, \
273  INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
274  INSTR_PROF_NAME_COFF, "__DATA,")
275 INSTR_PROF_SECT_ENTRY(IPSK_vals, \
276  INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
277  INSTR_PROF_VALS_COFF, "__DATA,")
278 INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
279  INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
280  INSTR_PROF_VNODES_COFF, "__DATA,")
281 INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
282  INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
283  INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
284 INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
285  INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
286  INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
287 INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
288  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
289  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
290 
291 #undef INSTR_PROF_SECT_ENTRY
292 #endif
293 
294 
295 #ifdef INSTR_PROF_VALUE_PROF_DATA
296 #define INSTR_PROF_DATA_DEFINED
297 
298 #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
299 /*!
300  * This is the header of the data structure that defines the on-disk
301  * layout of the value profile data of a particular kind for one function.
302  */
303 typedef struct ValueProfRecord {
304  /* The kind of the value profile record. */
305  uint32_t Kind;
306  /*
307  * The number of value profile sites. It is guaranteed to be non-zero;
308  * otherwise the record for this kind won't be emitted.
309  */
310  uint32_t NumValueSites;
311  /*
312  * The first element of the array that stores the number of profiled
313  * values for each value site. The size of the array is NumValueSites.
314  * Since NumValueSites is greater than zero, there is at least one
315  * element in the array.
316  */
317  uint8_t SiteCountArray[1];
318 
319  /*
320  * The fake declaration is for documentation purpose only.
321  * Align the start of next field to be on 8 byte boundaries.
322  uint8_t Padding[X];
323  */
324 
325  /* The array of value profile data. The size of the array is the sum
326  * of all elements in SiteCountArray[].
327  InstrProfValueData ValueData[];
328  */
329 
330 #ifdef __cplusplus
331  /*!
332  * Return the number of value sites.
333  */
334  uint32_t getNumValueSites() const { return NumValueSites; }
335  /*!
336  * Read data from this record and save it to Record.
337  */
338  void deserializeTo(InstrProfRecord &Record,
339  InstrProfSymtab *SymTab);
340  /*
341  * In-place byte swap:
342  * Do byte swap for this instance. \c Old is the original order before
343  * the swap, and \c New is the New byte order.
344  */
345  void swapBytes(support::endianness Old, support::endianness New);
346 #endif
347 } ValueProfRecord;
348 
349 /*!
350  * Per-function header/control data structure for value profiling
351  * data in indexed format.
352  */
353 typedef struct ValueProfData {
354  /*
355  * Total size in bytes including this field. It must be a multiple
356  * of sizeof(uint64_t).
357  */
358  uint32_t TotalSize;
359  /*
360  *The number of value profile kinds that has value profile data.
361  * In this implementation, a value profile kind is considered to
362  * have profile data if the number of value profile sites for the
363  * kind is not zero. More aggressively, the implementation can
364  * choose to check the actual data value: if none of the value sites
365  * has any profiled values, the kind can be skipped.
366  */
367  uint32_t NumValueKinds;
368 
369  /*
370  * Following are a sequence of variable length records. The prefix/header
371  * of each record is defined by ValueProfRecord type. The number of
372  * records is NumValueKinds.
373  * ValueProfRecord Record_1;
374  * ValueProfRecord Record_N;
375  */
376 
377 #if __cplusplus
378  /*!
379  * Return the total size in bytes of the on-disk value profile data
380  * given the data stored in Record.
381  */
382  static uint32_t getSize(const InstrProfRecord &Record);
383  /*!
384  * Return a pointer to \c ValueProfData instance ready to be streamed.
385  */
386  static std::unique_ptr<ValueProfData>
387  serializeFrom(const InstrProfRecord &Record);
388  /*!
389  * Check the integrity of the record.
390  */
391  Error checkIntegrity();
392  /*!
393  * Return a pointer to \c ValueProfileData instance ready to be read.
394  * All data in the instance are properly byte swapped. The input
395  * data is assumed to be in little endian order.
396  */
397  static Expected<std::unique_ptr<ValueProfData>>
398  getValueProfData(const unsigned char *SrcBuffer,
399  const unsigned char *const SrcBufferEnd,
400  support::endianness SrcDataEndianness);
401  /*!
402  * Swap byte order from \c Endianness order to host byte order.
403  */
404  void swapBytesToHost(support::endianness Endianness);
405  /*!
406  * Swap byte order from host byte order to \c Endianness order.
407  */
408  void swapBytesFromHost(support::endianness Endianness);
409  /*!
410  * Return the total size of \c ValueProfileData.
411  */
412  uint32_t getSize() const { return TotalSize; }
413  /*!
414  * Read data from this data and save it to \c Record.
415  */
416  void deserializeTo(InstrProfRecord &Record,
417  InstrProfSymtab *SymTab);
418  void operator delete(void *ptr) { ::operator delete(ptr); }
419 #endif
420 } ValueProfData;
421 
422 /*
423  * The closure is designed to abstact away two types of value profile data:
424  * - InstrProfRecord which is the primary data structure used to
425  * represent profile data in host tools (reader, writer, and profile-use)
426  * - value profile runtime data structure suitable to be used by C
427  * runtime library.
428  *
429  * Both sources of data need to serialize to disk/memory-buffer in common
430  * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
431  * writer to share the same format and code with indexed profile writer.
432  *
433  * For documentation of the member methods below, refer to corresponding methods
434  * in class InstrProfRecord.
435  */
436 typedef struct ValueProfRecordClosure {
437  const void *Record;
438  uint32_t (*GetNumValueKinds)(const void *Record);
439  uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
440  uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
441  uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
442 
443  /*
444  * After extracting the value profile data from the value profile record,
445  * this method is used to map the in-memory value to on-disk value. If
446  * the method is null, value will be written out untranslated.
447  */
448  uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
449  void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
450  uint32_t S);
451  ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
452 } ValueProfRecordClosure;
453 
454 INSTR_PROF_VISIBILITY ValueProfRecord *
455 getFirstValueProfRecord(ValueProfData *VPD);
456 INSTR_PROF_VISIBILITY ValueProfRecord *
457 getValueProfRecordNext(ValueProfRecord *VPR);
458 INSTR_PROF_VISIBILITY InstrProfValueData *
459 getValueProfRecordValueData(ValueProfRecord *VPR);
460 INSTR_PROF_VISIBILITY uint32_t
461 getValueProfRecordHeaderSize(uint32_t NumValueSites);
462 
463 #undef INSTR_PROF_VALUE_PROF_DATA
464 #endif /* INSTR_PROF_VALUE_PROF_DATA */
465 
466 
467 #ifdef INSTR_PROF_COMMON_API_IMPL
468 #define INSTR_PROF_DATA_DEFINED
469 #ifdef __cplusplus
470 #define INSTR_PROF_INLINE inline
471 #define INSTR_PROF_NULLPTR nullptr
472 #else
473 #define INSTR_PROF_INLINE
474 #define INSTR_PROF_NULLPTR NULL
475 #endif
476 
477 #ifndef offsetof
478 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
479 #endif
480 
481 /*!
482  * Return the \c ValueProfRecord header size including the
483  * padding bytes.
484  */
485 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
486 uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
487  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
488  sizeof(uint8_t) * NumValueSites;
489  /* Round the size to multiple of 8 bytes. */
490  Size = (Size + 7) & ~7;
491  return Size;
492 }
493 
494 /*!
495  * Return the total size of the value profile record including the
496  * header and the value data.
497  */
498 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
499 uint32_t getValueProfRecordSize(uint32_t NumValueSites,
500  uint32_t NumValueData) {
501  return getValueProfRecordHeaderSize(NumValueSites) +
502  sizeof(InstrProfValueData) * NumValueData;
503 }
504 
505 /*!
506  * Return the pointer to the start of value data array.
507  */
508 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
509 InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
510  return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
511  This->NumValueSites));
512 }
513 
514 /*!
515  * Return the total number of value data for \c This record.
516  */
517 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
518 uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
519  uint32_t NumValueData = 0;
520  uint32_t I;
521  for (I = 0; I < This->NumValueSites; I++)
522  NumValueData += This->SiteCountArray[I];
523  return NumValueData;
524 }
525 
526 /*!
527  * Use this method to advance to the next \c This \c ValueProfRecord.
528  */
529 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
530 ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
531  uint32_t NumValueData = getValueProfRecordNumValueData(This);
532  return (ValueProfRecord *)((char *)This +
533  getValueProfRecordSize(This->NumValueSites,
534  NumValueData));
535 }
536 
537 /*!
538  * Return the first \c ValueProfRecord instance.
539  */
540 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
541 ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
542  return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
543 }
544 
545 /* Closure based interfaces. */
546 
547 /*!
548  * Return the total size in bytes of the on-disk value profile data
549  * given the data stored in Record.
550  */
551 INSTR_PROF_VISIBILITY uint32_t
552 getValueProfDataSize(ValueProfRecordClosure *Closure) {
553  uint32_t Kind;
554  uint32_t TotalSize = sizeof(ValueProfData);
555  const void *Record = Closure->Record;
556 
557  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
558  uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
559  if (!NumValueSites)
560  continue;
561  TotalSize += getValueProfRecordSize(NumValueSites,
562  Closure->GetNumValueData(Record, Kind));
563  }
564  return TotalSize;
565 }
566 
567 /*!
568  * Extract value profile data of a function for the profile kind \c ValueKind
569  * from the \c Closure and serialize the data into \c This record instance.
570  */
571 INSTR_PROF_VISIBILITY void
572 serializeValueProfRecordFrom(ValueProfRecord *This,
573  ValueProfRecordClosure *Closure,
574  uint32_t ValueKind, uint32_t NumValueSites) {
575  uint32_t S;
576  const void *Record = Closure->Record;
577  This->Kind = ValueKind;
578  This->NumValueSites = NumValueSites;
579  InstrProfValueData *DstVD = getValueProfRecordValueData(This);
580 
581  for (S = 0; S < NumValueSites; S++) {
582  uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
583  This->SiteCountArray[S] = ND;
584  Closure->GetValueForSite(Record, DstVD, ValueKind, S);
585  DstVD += ND;
586  }
587 }
588 
589 /*!
590  * Extract value profile data of a function from the \c Closure
591  * and serialize the data into \c DstData if it is not NULL or heap
592  * memory allocated by the \c Closure's allocator method. If \c
593  * DstData is not null, the caller is expected to set the TotalSize
594  * in DstData.
595  */
596 INSTR_PROF_VISIBILITY ValueProfData *
597 serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
598  ValueProfData *DstData) {
599  uint32_t Kind;
600  uint32_t TotalSize =
601  DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
602 
603  ValueProfData *VPD =
604  DstData ? DstData : Closure->AllocValueProfData(TotalSize);
605 
606  VPD->TotalSize = TotalSize;
607  VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
608  ValueProfRecord *VR = getFirstValueProfRecord(VPD);
609  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
610  uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
611  if (!NumValueSites)
612  continue;
613  serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
614  VR = getValueProfRecordNext(VR);
615  }
616  return VPD;
617 }
618 
619 #undef INSTR_PROF_COMMON_API_IMPL
620 #endif /* INSTR_PROF_COMMON_API_IMPL */
621 
622 /*============================================================================*/
623 
624 #ifndef INSTR_PROF_DATA_DEFINED
625 
626 #ifndef INSTR_PROF_DATA_INC
627 #define INSTR_PROF_DATA_INC
628 
629 /* Helper macros. */
630 #define INSTR_PROF_SIMPLE_QUOTE(x) #x
631 #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
632 #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
633 #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
634 
635 /* Magic number to detect file format and endianness.
636  * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
637  * so that utilities, like strings, don't grab it as a string. 129 is also
638  * invalid UTF-8, and high enough to be interesting.
639  * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
640  * for 32-bit platforms.
641  */
642 #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
643  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
644  (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
645 #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
646  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
647  (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
648 
649 /* FIXME: Please remedy the fixme in the header before bumping the version. */
650 /* Raw profile format version (start from 1). */
651 #define INSTR_PROF_RAW_VERSION 8
652 /* Indexed profile format version (start from 1). */
653 #define INSTR_PROF_INDEX_VERSION 7
654 /* Coverage mapping format version (start from 0). */
655 #define INSTR_PROF_COVMAP_VERSION 5
656 
657 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
658  * version for other variants of profile. We set the lowest bit of the upper 8
659  * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
660  * generated profile, and 0 if this is a Clang FE generated profile.
661  * 1 in bit 57 indicates there are context-sensitive records in the profile.
662  * The 59th bit indicates whether to use debug info to correlate profiles.
663  */
664 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
665 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
666 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
667 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
668 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
669 #define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
670 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
671 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
672 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
673 
674 /* The variable that holds the name of the profile data
675  * specified via command line. */
676 #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
677 
678 /* section name strings common to all targets other
679  than WIN32 */
680 #define INSTR_PROF_DATA_COMMON __llvm_prf_data
681 #define INSTR_PROF_NAME_COMMON __llvm_prf_names
682 #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
683 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
684 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
685 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
686 #define INSTR_PROF_COVFUN_COMMON __llvm_covfun
687 #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
688 /* Windows section names. Because these section names contain dollar characters,
689  * they must be quoted.
690  */
691 #define INSTR_PROF_DATA_COFF ".lprfd$M"
692 #define INSTR_PROF_NAME_COFF ".lprfn$M"
693 #define INSTR_PROF_CNTS_COFF ".lprfc$M"
694 #define INSTR_PROF_VALS_COFF ".lprfv$M"
695 #define INSTR_PROF_VNODES_COFF ".lprfnd$M"
696 #define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
697 #define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
698 #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
699 
700 #ifdef _WIN32
701 /* Runtime section names and name strings. */
702 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
703 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
704 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
705 /* Array of pointers. Each pointer points to a list
706  * of value nodes associated with one value site.
707  */
708 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
709 /* Value profile nodes section. */
710 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
711 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
712 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
713 #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
714 #else
715 /* Runtime section names and name strings. */
716 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
717 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
718 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
719 /* Array of pointers. Each pointer points to a list
720  * of value nodes associated with one value site.
721  */
722 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
723 /* Value profile nodes section. */
724 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
725 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
726 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
727 /* Order file instrumentation. */
728 #define INSTR_PROF_ORDERFILE_SECT_NAME \
729  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
730 #endif
731 
732 #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
733 #define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
734  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
735 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
736 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
737  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
738 
739 /* Macros to define start/stop section symbol for a given
740  * section on Linux. For instance
741  * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
742  * expand to __start___llvm_prof_data
743  */
744 #define INSTR_PROF_SECT_START(Sect) \
745  INSTR_PROF_CONCAT(__start_,Sect)
746 #define INSTR_PROF_SECT_STOP(Sect) \
747  INSTR_PROF_CONCAT(__stop_,Sect)
748 
749 /* Value Profiling API linkage name. */
750 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
751 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
752  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
753 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
754 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
755  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
756 
757 /* InstrProfile per-function control data alignment. */
758 #define INSTR_PROF_DATA_ALIGNMENT 8
759 
760 /* The data structure that represents a tracked value by the
761  * value profiler.
762  */
763 typedef struct InstrProfValueData {
764  /* Profiled value. */
765  uint64_t Value;
766  /* Number of times the value appears in the training run. */
767  uint64_t Count;
768 } InstrProfValueData;
769 
770 #endif /* INSTR_PROF_DATA_INC */
771 
772 #ifndef INSTR_ORDER_FILE_INC
773 /* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
774 #define INSTR_ORDER_FILE_BUFFER_SIZE 131072
775 #define INSTR_ORDER_FILE_BUFFER_BITS 17
776 #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
777 #endif /* INSTR_ORDER_FILE_INC */
778 #else
779 #undef INSTR_PROF_DATA_DEFINED
780 #endif
781 
782 #undef COVMAP_V2_OR_V3
783 
784 #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
785 
786 #ifdef __cplusplus
787 #define INSTR_PROF_INLINE inline
788 #else
789 #define INSTR_PROF_INLINE
790 #endif
791 
792 /* The value range buckets (22 buckets) for the memop size value profiling looks
793  * like:
794  *
795  * [0, 0]
796  * [1, 1]
797  * [2, 2]
798  * [3, 3]
799  * [4, 4]
800  * [5, 5]
801  * [6, 6]
802  * [7, 7]
803  * [8, 8]
804  * [9, 15]
805  * [16, 16]
806  * [17, 31]
807  * [32, 32]
808  * [33, 63]
809  * [64, 64]
810  * [65, 127]
811  * [128, 128]
812  * [129, 255]
813  * [256, 256]
814  * [257, 511]
815  * [512, 512]
816  * [513, UINT64_MAX]
817  *
818  * Each range has a 'representative value' which is the lower end value of the
819  * range and used to store in the runtime profile data records and the VP
820  * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
821  */
822 #define INSTR_PROF_NUM_BUCKETS 22
823 
824 /*
825  * Clz and Popcount. This code was copied from
826  * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
827  * llvm/include/llvm/Support/MathExtras.h.
828  */
829 #if defined(_MSC_VER) && !defined(__clang__)
830 
831 #include <intrin.h>
832 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
833 int InstProfClzll(unsigned long long X) {
834  unsigned long LeadZeroIdx = 0;
835 #if !defined(_M_ARM64) && !defined(_M_X64)
836  // Scan the high 32 bits.
837  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
838  return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
839  // from the MSB.
840  // Scan the low 32 bits.
841  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
842  return (int)(63 - LeadZeroIdx);
843 #else
844  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
845 #endif
846  return 64;
847 }
848 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
849 int InstProfPopcountll(unsigned long long X) {
850  // This code originates from https://reviews.llvm.org/rG30626254510f.
851  unsigned long long v = X;
852  v = v - ((v >> 1) & 0x5555555555555555ULL);
853  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
854  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
855  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
856 }
857 
858 #else
859 
860 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
861 int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
862 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
863 int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
864 
865 #endif /* defined(_MSC_VER) && !defined(__clang__) */
866 
867 /* Map an (observed) memop size value to the representative value of its range.
868  * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
869 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
870 InstrProfGetRangeRepValue(uint64_t Value) {
871  if (Value <= 8)
872  // The first ranges are individually tracked. Use the value as is.
873  return Value;
874  else if (Value >= 513)
875  // The last range is mapped to its lowest value.
876  return 513;
877  else if (InstProfPopcountll(Value) == 1)
878  // If it's a power of two, use it as is.
879  return Value;
880  else
881  // Otherwise, take to the previous power of two + 1.
882  return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
883 }
884 
885 /* Return true if the range that an (observed) memop size value belongs to has
886  * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
887  * false, 64 -> true, 100 -> false, 513 -> false. */
888 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
889 InstrProfIsSingleValRange(uint64_t Value) {
890  if (Value <= 8)
891  // The first ranges are individually tracked.
892  return 1;
893  else if (InstProfPopcountll(Value) == 1)
894  // If it's a power of two, there's only one value.
895  return 1;
896  else
897  // Otherwise, there's more than one value in the range.
898  return 0;
899 }
900 
901 #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
VALUE_PROF_FUNC_PARAM
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType)
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
functions
amdgpu propagate attributes Late propagate attributes from kernels to functions
Definition: AMDGPUPropagateAttributes.cpp:196
INSTR_PROF_DATA
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init)
Definition: InstrProf.h:1134
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:23
llvm::msgpack::Endianness
constexpr support::endianness Endianness
The endianness of all multi-byte encoded values in MessagePack.
Definition: MsgPack.h:24
llvm::lltok::Error
@ Error
Definition: LLToken.h:21
endianness
llvm::support::endianness endianness
Definition: BinaryStreamReader.cpp:16
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:293
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:483
offsetof
#define offsetof(TYPE, MEMBER)
Definition: AMDHSAKernelDescriptor.h:23
llvm::IndexedInstrProf::ComputeHash
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1003
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2260
llvm::COFF::NameSize
@ NameSize
Definition: COFF.h:58
intrinsic
QP Compare Ordered outs ins xscmpudp No intrinsic
Definition: README_P9.txt:303
INSTR_PROF_RAW_HEADER
#define INSTR_PROF_RAW_HEADER(Type, Name, Init)
Definition: InstrProf.h:1143
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:239
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:241
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:207
COVMAP_FUNC_RECORD
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init)
Definition: CoverageMapping.h:918
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:73
llvm::coverage::CurrentVersion
@ CurrentVersion
Definition: CoverageMapping.h:1009
VALUE_PROF_KIND
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
Definition: InstrProf.h:239
COVMAP_HEADER
#define COVMAP_HEADER(Type, LLVMType, Name, Init)
Definition: CoverageMapping.h:971
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:932
llvm::ConstantDataArray::getRaw
static Constant * getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy)
getRaw() constructor - Return a constant with array type with an element count and element type match...
Definition: Constants.h:710
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
INSTR_PROF_VALUE_NODE
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init)
uint64_t
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
__DATA
the multiplication has a latency of four as opposed to two cycles for the movl lea variant It appears gcc place string data with linkonce linkage in section coalesced instead of section __DATA
Definition: README.txt:260
I
#define I(x, y, z)
Definition: MD5.cpp:58
size
i< reg-> size
Definition: README.txt:166
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:640
llvm::msf::Magic
static const char Magic[]
Definition: MSFCommon.h:23
uint32_t
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:242
uint16_t
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
This
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical This
Definition: README.txt:418
Version
uint64_t Version
Definition: RawMemProfReader.cpp:25
llvm::msgpack::Type
Type
MessagePack types as defined in the standard, with the exception of Integer being divided into a sign...
Definition: MsgPackReader.h:49
INSTR_PROF_SECT_ENTRY
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)
Definition: InstrProf.h:58