61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
67#ifndef INSTR_PROF_DATA
68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
70#define INSTR_PROF_DATA_DEFINED
76 ConstantInt::get(
llvm::
Type::getInt64Ty(Ctx), \
77 Inc->getHash()->getZExtValue()))
88 ConstantInt::get(
llvm::
Type::getInt32Ty(Ctx), NumCounters))
90 ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
107#ifndef INSTR_PROF_VALUE_NODE
108#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
110#define INSTR_PROF_DATA_DEFINED
113 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
115 ConstantInt::get(
llvm::
Type::GetInt64Ty(Ctx), 0))
117 ConstantInt::get(
llvm::
Type::GetInt8PtrTy(Ctx), 0))
118#undef INSTR_PROF_VALUE_NODE
123#ifndef INSTR_PROF_RAW_HEADER
124#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
126#define INSTR_PROF_DATA_DEFINED
139 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
142#undef INSTR_PROF_RAW_HEADER
149#ifndef VALUE_PROF_FUNC_PARAM
150#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
151#define INSTR_PROF_COMMA
153#define INSTR_PROF_DATA_DEFINED
154#define INSTR_PROF_COMMA ,
160#undef VALUE_PROF_FUNC_PARAM
161#undef INSTR_PROF_COMMA
165#ifndef VALUE_PROF_KIND
166#define VALUE_PROF_KIND(Enumerator, Value, Descr)
168#define INSTR_PROF_DATA_DEFINED
190#undef VALUE_PROF_KIND
193#undef COVMAP_V2_OR_V3
195#define COVMAP_V2_OR_V3
198#define COVMAP_V2_OR_V3
205#ifndef COVMAP_FUNC_RECORD
206#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
208#define INSTR_PROF_DATA_DEFINED
218#ifdef COVMAP_V2_OR_V3
227 llvm::ConstantInt::get( \
228 llvm::
Type::getInt64Ty(Ctx), FuncHash))
235 CoverageMapping.size()), \
238 CoverageMapping, CoverageMapping.size(), \
241#undef COVMAP_FUNC_RECORD
248#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
250#define INSTR_PROF_DATA_DEFINED
264#ifdef INSTR_PROF_SECT_ENTRY
265#define INSTR_PROF_DATA_DEFINED
267 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
268 INSTR_PROF_DATA_COFF,
"__DATA,")
270 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
271 INSTR_PROF_CNTS_COFF, "__DATA,")
273 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
274 INSTR_PROF_NAME_COFF, "__DATA,")
276 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
277 INSTR_PROF_VALS_COFF, "__DATA,")
279 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
280 INSTR_PROF_VNODES_COFF, "__DATA,")
282 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
283 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
285 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
286 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
288 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
289 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
291#undef INSTR_PROF_SECT_ENTRY
295#ifdef INSTR_PROF_VALUE_PROF_DATA
296#define INSTR_PROF_DATA_DEFINED
298#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
303typedef struct ValueProfRecord {
317 uint8_t SiteCountArray[1];
334 uint32_t getNumValueSites()
const {
return NumValueSites; }
338 void deserializeTo(InstrProfRecord &Record,
339 InstrProfSymtab *SymTab);
345 void swapBytes(support::endianness Old, support::endianness New);
353typedef struct ValueProfData {
386 static std::unique_ptr<ValueProfData>
387 serializeFrom(
const InstrProfRecord &Record);
391 Error checkIntegrity();
397 static Expected<std::unique_ptr<ValueProfData>>
398 getValueProfData(
const unsigned char *SrcBuffer,
399 const unsigned char *
const SrcBufferEnd,
400 support::endianness SrcDataEndianness);
404 void swapBytesToHost(support::endianness Endianness);
408 void swapBytesFromHost(support::endianness Endianness);
416 void deserializeTo(InstrProfRecord &Record,
417 InstrProfSymtab *SymTab);
418 void operator delete(
void *ptr) { ::operator
delete(ptr); }
436typedef struct ValueProfRecordClosure {
438 uint32_t (*GetNumValueKinds)(
const void *Record);
449 void (*GetValueForSite)(
const void *
R, InstrProfValueData *Dst,
uint32_t K,
451 ValueProfData *(*AllocValueProfData)(
size_t TotalSizeInBytes);
452} ValueProfRecordClosure;
454INSTR_PROF_VISIBILITY ValueProfRecord *
455getFirstValueProfRecord(ValueProfData *VPD);
456INSTR_PROF_VISIBILITY ValueProfRecord *
457getValueProfRecordNext(ValueProfRecord *VPR);
458INSTR_PROF_VISIBILITY InstrProfValueData *
459getValueProfRecordValueData(ValueProfRecord *VPR);
461getValueProfRecordHeaderSize(
uint32_t NumValueSites);
463#undef INSTR_PROF_VALUE_PROF_DATA
467#ifdef INSTR_PROF_COMMON_API_IMPL
468#define INSTR_PROF_DATA_DEFINED
470#define INSTR_PROF_INLINE inline
471#define INSTR_PROF_NULLPTR nullptr
473#define INSTR_PROF_INLINE
474#define INSTR_PROF_NULLPTR NULL
478#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
485INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
488 sizeof(uint8_t) * NumValueSites;
498INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
501 return getValueProfRecordHeaderSize(NumValueSites) +
502 sizeof(InstrProfValueData) * NumValueData;
508INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
509InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
510 return (InstrProfValueData *)((
char *)This + getValueProfRecordHeaderSize(
511 This->NumValueSites));
517INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
518uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
521 for (
I = 0;
I <
This->NumValueSites;
I++)
522 NumValueData +=
This->SiteCountArray[
I];
529INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
530ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
531 uint32_t NumValueData = getValueProfRecordNumValueData(This);
532 return (ValueProfRecord *)((
char *)This +
533 getValueProfRecordSize(
This->NumValueSites,
540INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
541ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
542 return (ValueProfRecord *)((
char *)This +
sizeof(ValueProfData));
552getValueProfDataSize(ValueProfRecordClosure *Closure) {
554 uint32_t TotalSize =
sizeof(ValueProfData);
555 const void *Record = Closure->Record;
557 for (Kind = IPVK_First;
Kind <= IPVK_Last;
Kind++) {
558 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
561 TotalSize += getValueProfRecordSize(NumValueSites,
562 Closure->GetNumValueData(Record, Kind));
571INSTR_PROF_VISIBILITY
void
572serializeValueProfRecordFrom(ValueProfRecord *This,
573 ValueProfRecordClosure *Closure,
576 const void *Record = Closure->Record;
578 This->NumValueSites = NumValueSites;
579 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
581 for (S = 0; S < NumValueSites; S++) {
582 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
583 This->SiteCountArray[S] = ND;
584 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
596INSTR_PROF_VISIBILITY ValueProfData *
597serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
598 ValueProfData *DstData) {
601 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
604 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
606 VPD->TotalSize = TotalSize;
607 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
608 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
609 for (Kind = IPVK_First;
Kind <= IPVK_Last;
Kind++) {
610 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
613 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
614 VR = getValueProfRecordNext(VR);
619#undef INSTR_PROF_COMMON_API_IMPL
624#ifndef INSTR_PROF_DATA_DEFINED
626#ifndef INSTR_PROF_DATA_INC
627#define INSTR_PROF_DATA_INC
630#define INSTR_PROF_SIMPLE_QUOTE(x) #x
631#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
632#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
633#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
642#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
643 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
644 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
645#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
646 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
647 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
651#define INSTR_PROF_RAW_VERSION 8
653#define INSTR_PROF_INDEX_VERSION 9
655#define INSTR_PROF_COVMAP_VERSION 5
667#define VARIANT_MASKS_ALL 0xff00000000000000ULL
668#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
669#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
670#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
671#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
672#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
673#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
674#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
675#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
676#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
677#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
678#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
682#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
686#define INSTR_PROF_DATA_COMMON __llvm_prf_data
687#define INSTR_PROF_NAME_COMMON __llvm_prf_names
688#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
689#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
690#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
691#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
692#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
693#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
697#define INSTR_PROF_DATA_COFF ".lprfd$M"
698#define INSTR_PROF_NAME_COFF ".lprfn$M"
699#define INSTR_PROF_CNTS_COFF ".lprfc$M"
700#define INSTR_PROF_VALS_COFF ".lprfv$M"
701#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
702#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
703#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
704#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
708#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
709#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
710#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
714#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
716#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
717#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
718#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
719#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
722#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
723#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
724#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
728#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
730#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
731#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
732#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
734#define INSTR_PROF_ORDERFILE_SECT_NAME \
735 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
738#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
739#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
740 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
741#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
742#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
743 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
750#define INSTR_PROF_SECT_START(Sect) \
751 INSTR_PROF_CONCAT(__start_,Sect)
752#define INSTR_PROF_SECT_STOP(Sect) \
753 INSTR_PROF_CONCAT(__stop_,Sect)
756#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
757#define INSTR_PROF_VALUE_PROF_FUNC_STR \
758 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
759#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
760#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
761 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
764#define INSTR_PROF_DATA_ALIGNMENT 8
769typedef struct InstrProfValueData {
778#ifndef INSTR_ORDER_FILE_INC
780#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
781#define INSTR_ORDER_FILE_BUFFER_BITS 17
782#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
785#undef INSTR_PROF_DATA_DEFINED
788#undef COVMAP_V2_OR_V3
790#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
793#define INSTR_PROF_INLINE inline
795#define INSTR_PROF_INLINE
828#define INSTR_PROF_NUM_BUCKETS 22
835#if defined(_MSC_VER) && !defined(__clang__)
838INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
839int InstProfClzll(
unsigned long long X) {
840 unsigned long LeadZeroIdx = 0;
841#if !defined(_M_ARM64) && !defined(_M_X64)
843 if (_BitScanReverse(&LeadZeroIdx, (
unsigned long)(
X >> 32)))
844 return (
int)(63 - (LeadZeroIdx + 32));
847 if (_BitScanReverse(&LeadZeroIdx, (
unsigned long)(
X)))
848 return (
int)(63 - LeadZeroIdx);
850 if (_BitScanReverse64(&LeadZeroIdx,
X))
return 63 - LeadZeroIdx;
854INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
855int InstProfPopcountll(
unsigned long long X) {
857 unsigned long long v =
X;
858 v =
v - ((
v >> 1) & 0x5555555555555555ULL);
859 v = (
v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
860 v = (
v + (
v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
861 return (
int)((
unsigned long long)(v * 0x0101010101010101ULL) >> 56);
866INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
867int InstProfClzll(
unsigned long long X) {
return __builtin_clzll(
X); }
868INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
869int InstProfPopcountll(
unsigned long long X) {
return __builtin_popcountll(
X); }
875INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
uint64_t
876InstrProfGetRangeRepValue(
uint64_t Value) {
880 else if (Value >= 513)
883 else if (InstProfPopcountll(Value) == 1)
888 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
894INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
unsigned
895InstrProfIsSingleValRange(
uint64_t Value) {
899 else if (InstProfPopcountll(Value) == 1)
amdgpu propagate attributes Late propagate attributes from kernels to functions
#define offsetof(TYPE, MEMBER)
#define COVMAP_HEADER(Type, LLVMType, Name, Init)
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
#define INSTR_PROF_RAW_HEADER(Type, Name, Init)
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init)
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init)
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType)
return ToRemove size() > 0
static unsigned getSize(unsigned Kind)
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
static Constant * getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy)
getRaw() constructor - Return a constant with array type with an element count and element type match...
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static IntegerType * getInt8Ty(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
This is an optimization pass for GlobalISel generic memory operations.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.