61 #ifndef INSTR_PROF_VISIBILITY
62 #define INSTR_PROF_VISIBILITY
67 #ifndef INSTR_PROF_DATA
68 #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
70 #define INSTR_PROF_DATA_DEFINED
77 Inc->getHash()->getZExtValue()))
88 ConstantInt::
get(
llvm::
Type::getInt32Ty(Ctx), NumCounters))
90 ConstantArray::
get(Int16ArrayTy, Int16ArrayVals))
91 #undef INSTR_PROF_DATA
107 #ifndef INSTR_PROF_VALUE_NODE
108 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
110 #define INSTR_PROF_DATA_DEFINED
117 ConstantInt::
get(
llvm::
Type::GetInt8PtrTy(Ctx), 0))
118 #undef INSTR_PROF_VALUE_NODE
123 #ifndef INSTR_PROF_RAW_HEADER
124 #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
126 #define INSTR_PROF_DATA_DEFINED
139 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
142 #undef INSTR_PROF_RAW_HEADER
149 #ifndef VALUE_PROF_FUNC_PARAM
150 #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
151 #define INSTR_PROF_COMMA
153 #define INSTR_PROF_DATA_DEFINED
154 #define INSTR_PROF_COMMA ,
160 #undef VALUE_PROF_FUNC_PARAM
161 #undef INSTR_PROF_COMMA
165 #ifndef VALUE_PROF_KIND
166 #define VALUE_PROF_KIND(Enumerator, Value, Descr)
168 #define INSTR_PROF_DATA_DEFINED
190 #undef VALUE_PROF_KIND
193 #undef COVMAP_V2_OR_V3
195 #define COVMAP_V2_OR_V3
198 #define COVMAP_V2_OR_V3
205 #ifndef COVMAP_FUNC_RECORD
206 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
208 #define INSTR_PROF_DATA_DEFINED
218 #ifdef COVMAP_V2_OR_V3
228 llvm::
Type::getInt64Ty(Ctx), FuncHash))
235 CoverageMapping.size()), \
238 CoverageMapping, CoverageMapping.size(), \
241 #undef COVMAP_FUNC_RECORD
247 #ifndef COVMAP_HEADER
248 #define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
250 #define INSTR_PROF_DATA_DEFINED
264 #ifdef INSTR_PROF_SECT_ENTRY
265 #define INSTR_PROF_DATA_DEFINED
267 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
268 INSTR_PROF_DATA_COFF,
"__DATA,")
270 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
271 INSTR_PROF_CNTS_COFF, "
__DATA,")
273 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
274 INSTR_PROF_NAME_COFF, "
__DATA,")
276 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
277 INSTR_PROF_VALS_COFF, "
__DATA,")
279 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
280 INSTR_PROF_VNODES_COFF, "
__DATA,")
282 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
283 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
285 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
286 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
288 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
289 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "
__DATA,")
291 #undef INSTR_PROF_SECT_ENTRY
295 #ifdef INSTR_PROF_VALUE_PROF_DATA
296 #define INSTR_PROF_DATA_DEFINED
298 #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
303 typedef struct ValueProfRecord {
317 uint8_t SiteCountArray[1];
334 uint32_t getNumValueSites()
const {
return NumValueSites; }
338 void deserializeTo(InstrProfRecord &Record,
339 InstrProfSymtab *SymTab);
353 typedef struct ValueProfData {
382 static uint32_t getSize(
const InstrProfRecord &Record);
386 static std::unique_ptr<ValueProfData>
387 serializeFrom(
const InstrProfRecord &Record);
391 Error checkIntegrity();
397 static Expected<std::unique_ptr<ValueProfData>>
398 getValueProfData(
const unsigned char *SrcBuffer,
399 const unsigned char *
const SrcBufferEnd,
412 uint32_t getSize()
const {
return TotalSize; }
416 void deserializeTo(InstrProfRecord &Record,
417 InstrProfSymtab *SymTab);
418 void operator delete(
void *ptr) { ::operator
delete(ptr); }
436 typedef struct ValueProfRecordClosure {
438 uint32_t (*GetNumValueKinds)(
const void *Record);
449 void (*GetValueForSite)(
const void *
R, InstrProfValueData *Dst,
uint32_t K,
451 ValueProfData *(*AllocValueProfData)(
size_t TotalSizeInBytes);
452 } ValueProfRecordClosure;
454 INSTR_PROF_VISIBILITY ValueProfRecord *
455 getFirstValueProfRecord(ValueProfData *VPD);
456 INSTR_PROF_VISIBILITY ValueProfRecord *
457 getValueProfRecordNext(ValueProfRecord *VPR);
458 INSTR_PROF_VISIBILITY InstrProfValueData *
459 getValueProfRecordValueData(ValueProfRecord *VPR);
461 getValueProfRecordHeaderSize(
uint32_t NumValueSites);
463 #undef INSTR_PROF_VALUE_PROF_DATA
467 #ifdef INSTR_PROF_COMMON_API_IMPL
468 #define INSTR_PROF_DATA_DEFINED
470 #define INSTR_PROF_INLINE inline
471 #define INSTR_PROF_NULLPTR nullptr
473 #define INSTR_PROF_INLINE
474 #define INSTR_PROF_NULLPTR NULL
478 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
485 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
488 sizeof(uint8_t) * NumValueSites;
498 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
501 return getValueProfRecordHeaderSize(NumValueSites) +
502 sizeof(InstrProfValueData) * NumValueData;
508 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
509 InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *
This) {
510 return (InstrProfValueData *)((
char *)
This + getValueProfRecordHeaderSize(
511 This->NumValueSites));
517 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
518 uint32_t getValueProfRecordNumValueData(ValueProfRecord *
This) {
521 for (
I = 0;
I <
This->NumValueSites;
I++)
522 NumValueData +=
This->SiteCountArray[
I];
529 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
530 ValueProfRecord *getValueProfRecordNext(ValueProfRecord *
This) {
531 uint32_t NumValueData = getValueProfRecordNumValueData(
This);
532 return (ValueProfRecord *)((
char *)
This +
533 getValueProfRecordSize(
This->NumValueSites,
540 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
541 ValueProfRecord *getFirstValueProfRecord(ValueProfData *
This) {
542 return (ValueProfRecord *)((
char *)
This +
sizeof(ValueProfData));
552 getValueProfDataSize(ValueProfRecordClosure *Closure) {
554 uint32_t TotalSize =
sizeof(ValueProfData);
555 const void *Record = Closure->Record;
558 uint32_t NumValueSites = Closure->GetNumValueSites(Record,
Kind);
561 TotalSize += getValueProfRecordSize(NumValueSites,
562 Closure->GetNumValueData(Record,
Kind));
571 INSTR_PROF_VISIBILITY
void
572 serializeValueProfRecordFrom(ValueProfRecord *
This,
573 ValueProfRecordClosure *Closure,
576 const void *Record = Closure->Record;
578 This->NumValueSites = NumValueSites;
579 InstrProfValueData *DstVD = getValueProfRecordValueData(
This);
581 for (
S = 0;
S < NumValueSites;
S++) {
583 This->SiteCountArray[
S] = ND;
584 Closure->GetValueForSite(Record, DstVD,
ValueKind,
S);
596 INSTR_PROF_VISIBILITY ValueProfData *
597 serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
598 ValueProfData *DstData) {
601 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
604 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
606 VPD->TotalSize = TotalSize;
607 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
608 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
610 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record,
Kind);
613 serializeValueProfRecordFrom(VR, Closure,
Kind, NumValueSites);
614 VR = getValueProfRecordNext(VR);
619 #undef INSTR_PROF_COMMON_API_IMPL
624 #ifndef INSTR_PROF_DATA_DEFINED
626 #ifndef INSTR_PROF_DATA_INC
627 #define INSTR_PROF_DATA_INC
630 #define INSTR_PROF_SIMPLE_QUOTE(x) #x
631 #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
632 #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
633 #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
642 #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
643 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
644 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
645 #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
646 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
647 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
651 #define INSTR_PROF_RAW_VERSION 8
653 #define INSTR_PROF_INDEX_VERSION 8
655 #define INSTR_PROF_COVMAP_VERSION 5
667 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
668 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
669 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
670 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
671 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
672 #define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
673 #define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
674 #define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
675 #define VARIANT_MASK_MEMPROF (0x1ULL << 62)
676 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
677 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
678 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
682 #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
686 #define INSTR_PROF_DATA_COMMON __llvm_prf_data
687 #define INSTR_PROF_NAME_COMMON __llvm_prf_names
688 #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
689 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
690 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
691 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
692 #define INSTR_PROF_COVFUN_COMMON __llvm_covfun
693 #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
697 #define INSTR_PROF_DATA_COFF ".lprfd$M"
698 #define INSTR_PROF_NAME_COFF ".lprfn$M"
699 #define INSTR_PROF_CNTS_COFF ".lprfc$M"
700 #define INSTR_PROF_VALS_COFF ".lprfv$M"
701 #define INSTR_PROF_VNODES_COFF ".lprfnd$M"
702 #define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
703 #define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
704 #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
708 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
709 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
710 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
714 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
716 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
717 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
718 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
719 #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
722 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
723 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
724 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
728 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
730 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
731 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
732 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
734 #define INSTR_PROF_ORDERFILE_SECT_NAME \
735 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
738 #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
739 #define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
740 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
741 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
742 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
743 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
750 #define INSTR_PROF_SECT_START(Sect) \
751 INSTR_PROF_CONCAT(__start_,Sect)
752 #define INSTR_PROF_SECT_STOP(Sect) \
753 INSTR_PROF_CONCAT(__stop_,Sect)
756 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
757 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
758 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
759 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
760 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
761 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
764 #define INSTR_PROF_DATA_ALIGNMENT 8
769 typedef struct InstrProfValueData {
774 } InstrProfValueData;
778 #ifndef INSTR_ORDER_FILE_INC
780 #define INSTR_ORDER_FILE_BUFFER_SIZE 131072
781 #define INSTR_ORDER_FILE_BUFFER_BITS 17
782 #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
785 #undef INSTR_PROF_DATA_DEFINED
788 #undef COVMAP_V2_OR_V3
790 #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
793 #define INSTR_PROF_INLINE inline
795 #define INSTR_PROF_INLINE
828 #define INSTR_PROF_NUM_BUCKETS 22
835 #if defined(_MSC_VER) && !defined(__clang__)
838 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
839 int InstProfClzll(
unsigned long long X) {
840 unsigned long LeadZeroIdx = 0;
841 #if !defined(_M_ARM64) && !defined(_M_X64)
843 if (_BitScanReverse(&LeadZeroIdx, (
unsigned long)(
X >> 32)))
844 return (
int)(63 - (LeadZeroIdx + 32));
847 if (_BitScanReverse(&LeadZeroIdx, (
unsigned long)(
X)))
848 return (
int)(63 - LeadZeroIdx);
850 if (_BitScanReverse64(&LeadZeroIdx,
X))
return 63 - LeadZeroIdx;
854 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
855 int InstProfPopcountll(
unsigned long long X) {
857 unsigned long long v =
X;
858 v = v - ((v >> 1) & 0x5555555555555555ULL);
859 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
860 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
861 return (
int)((
unsigned long long)(v * 0x0101010101010101ULL) >> 56);
866 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
867 int InstProfClzll(
unsigned long long X) {
return __builtin_clzll(
X); }
868 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
869 int InstProfPopcountll(
unsigned long long X) {
return __builtin_popcountll(
X); }
875 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
uint64_t
880 else if (
Value >= 513)
883 else if (InstProfPopcountll(
Value) == 1)
888 return (UINT64_C(1) << (64 - InstProfClzll(
Value) - 1)) + 1;
894 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
unsigned
899 else if (InstProfPopcountll(
Value) == 1)