16 #ifndef LLVM_PROFILEDATA_INSTRPROF_H
17 #define LLVM_PROFILEDATA_INSTRPROF_H
32 #include <system_error>
43 return AddSegment ?
"__DATA," INSTR_PROF_CNTS_SECT_NAME_STR
44 : INSTR_PROF_CNTS_SECT_NAME_STR;
50 return AddSegment ?
"__DATA," INSTR_PROF_NAME_SECT_NAME_STR
51 : INSTR_PROF_NAME_SECT_NAME_STR;
57 return AddSegment ?
"__DATA," INSTR_PROF_DATA_SECT_NAME_STR
58 ",regular,live_support"
59 : INSTR_PROF_DATA_SECT_NAME_STR;
65 return AddSegment ?
"__DATA," INSTR_PROF_VALS_SECT_NAME_STR
66 : INSTR_PROF_VALS_SECT_NAME_STR;
72 return AddSegment ?
"__DATA," INSTR_PROF_VNODES_SECT_NAME_STR
73 : INSTR_PROF_VNODES_SECT_NAME_STR;
79 return INSTR_PROF_VALUE_PROF_FUNC_STR;
85 return AddSegment ?
"__LLVM_COV," INSTR_PROF_COVMAP_SECT_NAME_STR
86 : INSTR_PROF_COVMAP_SECT_NAME_STR;
111 return "__llvm_prf_nm";
118 return "__llvm_coverage_mapping";
126 return "__llvm_coverage_names";
135 return "__llvm_profile_register_functions";
141 return "__llvm_profile_register_function";
146 return "__llvm_profile_register_names_function";
159 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
165 return "__llvm_profile_runtime_user";
175 uint64_t
Version = INSTR_PROF_INDEX_VERSION);
184 uint64_t
Version = INSTR_PROF_INDEX_VERSION);
203 StringRef PGOFuncName);
210 StringRef FileName =
"<unknown>");
222 bool doCompression, std::string &Result);
226 std::string &Result,
bool doCompression =
true);
227 class InstrProfSymtab;
243 #define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value,
247 struct InstrProfRecord;
253 const InstrProfRecord &InstrProfR,
258 ArrayRef<InstrProfValueData> VDs,
268 InstrProfValueData ValueData[],
269 uint32_t &ActualNumValueData, uint64_t &TotalC);
318 std::string
message()
const override;
353 unsigned NumHashMismatches;
356 unsigned NumCountMismatches;
359 unsigned NumCounterOverflows;
362 unsigned NumValueSiteCountMismatches;
367 NumCountMismatches(0), NumCounterOverflows(0),
368 NumValueSiteCountMismatches(0) {}
372 "Unchecked soft error encountered");
389 return NumValueSiteCountMismatches;
397 auto E = make_error<InstrProfError>(FirstError);
407 namespace IndexedInstrProf {
426 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
429 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
436 : Data(), Address(0), NameTab(), MD5NameMap(), MD5FuncMap(),
460 template <
typename NameIterRange>
void create(
const NameIterRange &IterRange);
471 MD5NameMap.push_back(std::make_pair(
477 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
507 template <
typename NameIterRange>
509 for (
auto Name : IterRange)
516 std::sort(MD5NameMap.begin(), MD5NameMap.end(),
less_first());
517 std::sort(MD5FuncMap.begin(), MD5FuncMap.end(),
less_first());
518 std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(),
less_first());
519 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
525 std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash,
526 [](
const std::pair<uint64_t, std::string> &LHS,
527 uint64_t RHS) {
return LHS.first < RHS; });
528 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
529 return Result->second;
535 std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash,
536 [](
const std::pair<uint64_t, Function*> &LHS,
537 uint64_t RHS) {
return LHS.first < RHS; });
538 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
539 return Result->second;
558 template <
class InputIterator>
565 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
566 return left.Value < right.Value;
575 uint64_t Weight = 1);
584 : Name(Name), Hash(Hash), Counts(std::move(Counts)),
SIPE() {}
606 inline std::unique_ptr<InstrProfValueData[]>
608 uint64_t *TotalC = 0)
const;
627 void scale(uint64_t Weight);
632 std::vector<InstrProfValueSiteRecord> &SiteRecords =
633 getValueSitesForKind(
Kind);
634 for (
auto &SR : SiteRecords)
641 getValueSitesForKind(
Kind).clear();
648 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
649 const std::vector<InstrProfValueSiteRecord> &
650 getValueSitesForKind(
uint32_t ValueKind)
const {
652 case IPVK_IndirectCallTarget:
653 return IndirectCallSites;
657 return IndirectCallSites;
660 std::vector<InstrProfValueSiteRecord> &
661 getValueSitesForKind(
uint32_t ValueKind) {
662 return const_cast<std::vector<InstrProfValueSiteRecord> &
>(
664 ->getValueSitesForKind(ValueKind));
668 uint64_t remapValue(uint64_t Value,
uint32_t ValueKind,
676 void scaleValueProfData(
uint32_t ValueKind, uint64_t Weight);
682 NumValueKinds += !(getValueSitesForKind(
Kind).empty());
683 return NumValueKinds;
688 const std::vector<InstrProfValueSiteRecord> &SiteRecords =
689 getValueSitesForKind(ValueKind);
690 for (
auto &SR : SiteRecords) {
691 N += SR.ValueData.size();
697 return getValueSitesForKind(ValueKind).size();
702 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
705 std::unique_ptr<InstrProfValueData[]>
707 uint64_t *TotalC)
const {
709 uint64_t &TotalCount = (TotalC == 0 ? Dummy : *TotalC);
713 return std::unique_ptr<InstrProfValueData[]>(
nullptr);
716 auto VD = llvm::make_unique<InstrProfValueData[]>(
N);
726 uint64_t TotalCount = 0;
727 for (
auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
728 Dest[
I].Value = V.Value;
729 Dest[
I].Count = V.Count;
737 std::vector<InstrProfValueSiteRecord> &ValueSites =
738 getValueSitesForKind(ValueKind);
739 ValueSites.reserve(NumValueSites);
747 #define INSTR_PROF_VALUE_PROF_DATA
752 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
753 return left.Count > right.Count;
756 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
761 namespace IndexedInstrProf {
777 const uint64_t
Magic = 0x8169666f72706cff;
853 return sizeof(
Summary) + NumCutoffEntries *
sizeof(
Entry) +
854 NumSumFields *
sizeof(uint64_t);
858 return reinterpret_cast<const uint64_t *
>(
this + 1);
861 return reinterpret_cast<uint64_t *
>(
this + 1);
864 return reinterpret_cast<const Entry *
>(
888 void operator delete(
void *ptr) { ::operator
delete(ptr); }
894 return std::unique_ptr<Summary>(
new (::operator
new(TotalSize))
899 namespace RawInstrProf {
908 const uint64_t
Version = INSTR_PROF_RAW_VERSION;
910 template <
class IntPtrT>
inline uint64_t
getMagic();
912 return INSTR_PROF_RAW_MAGIC_64;
916 return INSTR_PROF_RAW_MAGIC_32;
925 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
934 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
942 #endif // LLVM_PROFILEDATA_INSTRPROF_H
void scale(uint64_t Weight)
Scale up profile counts (including value profile data) by Weight.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
void log(raw_ostream &OS) const override
Print an error message to an output stream.
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable...
unsigned getNumCountMismatches() const
Get the number of count mismatches.
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumCutoffEntries
void sortByCount()
Sort ValueData Descending by Count.
uint32_t Cutoff
The required percentile of counts.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
void sortValueData()
Sort value profile data (per site) by count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
StringRef getOrigFuncName(uint64_t FuncMD5Hash)
Return the function's original assembly name by stripping off the prefix attached (to symbols with pr...
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
A Module instance is used to store all the information related to an LLVM module. ...
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
const Entry & getEntry(uint32_t I) const
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
static instrprof_error take(Error E)
Consume an Error and return the raw enum value contained within it.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
uint64_t getMagic< uint64_t >()
Max block count of the program.
AddrHashMap & getAddrHashMap()
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
InstrProfValueSiteRecord()
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
uint64_t * getSummaryDataBase()
std::enable_if< std::is_unsigned< T >::value, T >::type SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
StringRef getInstrProfDataSectionName(bool AddSegment)
Return the name of the data section containing per-function control data.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error takeError()
Get the error contained within the record's soft error counter.
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
InstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
const std::error_category & instrprof_category()
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
std::error_code make_error_code(BitcodeError E)
StringRef getInstrProfCoverageSectionName(bool AddSegment)
Return the name of the section containing function coverage mapping data.
void merge(InstrProfRecord &Other, uint64_t Weight=1)
Merge the counts in Other into this one.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Error collectPGOFuncNameStrings(const std::vector< std::string > &NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
const Entry * getCutoffEntryBase() const
uint64_t NumSummaryFields
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
static const bool IsLittleEndianHost
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name...
unsigned getNumValueSiteCountMismatches() const
Get the number of value site count mismatches.
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
Entry * getCutoffEntryBase()
void addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
void merge(SoftInstrProfErrors &SIPE, InstrProfValueSiteRecord &Input, uint64_t Weight=1)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
std::string message() const override
Return the error message as a string.
uint64_t getMagic< uint32_t >()
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC)
Extract the value profile data from Inst which is annotated with value profile meta data...
void scale(SoftInstrProfErrors &SIPE, uint64_t Weight)
Scale up value profile data counts.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
uint64_t Cutoff
The required percentile of total execution count.
The instances of the Type class are immutable: once they are created, they are never changed...
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
static ManagedStatic< _object_error_category > error_category
StringRef getInstrProfValuesSectionName(bool AddSegment)
Return the name of data section containing pointers to value profile counters/nodes.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void handleAllErrors(Error E, HandlerTs &&...Handlers)
Behaves the same as handleErrors, except that it requires that all errors be handled by the given han...
uint64_t ComputeHash(StringRef K)
unsigned getNumHashMismatches() const
Get the number of hash mismatches.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above...
std::pair< typename base::iterator, bool > insert(StringRef Key)
Error takeError()
Return the first encountered error and reset FirstError to a success value.
struct LLVM_ALIGNAS(8) ProfileData
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, ValueMapType *ValueMap)
Add ValueData for ValueKind at value Site.
void clearValueData()
Clear value data entries.
static ErrorSuccess success()
Create a success value.
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
const uint64_t * getSummaryDataBase() const
instrprof_error get() const
StringRef getPGOFuncNameMetadataName()
StringRef getInstrProfVNodesSectionName(bool AddSegment)
Return the name of data section containing nodes holdling value profiling data.
LinkageTypes
An enumeration for the kinds of linkage for global values.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Base class for user error types.
void sortByTargetValues()
Sort ValueData ascending by Value.
The total number of functions instrumented.
std::vector< uint64_t > Counts
unsigned getNumCounterOverflows() const
Get the number of counter overflows.
StringRef getInstrProfCountersSectionName(bool AddSegment)
Return the name of data section containing profile counter variables.
The sum of all instrumented block counts.
Profiling information for a single function.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
The maximal execution count among all functions.
StringRef getNameData() const
Return the name section data.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
LLVM_NODISCARD size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Max internal block count of the program (excluding entry blocks).
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=0) const
Return the array of profiled values at Site.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Lightweight error class with error context and mandatory checking.
uint64_t MinCount
The minimum count for this percentile.
StringSet - A wrapper for StringMap that provides set-like functionality.
support::endianness getHostEndianness()
This class implements an extremely fast bulk output stream that can only output to a stream...
std::vector< std::pair< uint64_t, uint64_t > > ValueMapType
StringRef - Represent a constant reference to a string, i.e.
Total number of instrumented blocks/edges.
uint64_t NumCounts
Number of counts >= the minimum count.
StringRef getInstrProfComdatPrefix()
Return the name prefix of the COMDAT group for instrumentation variables associated with a COMDAT fun...
StringRef getInstrProfNameSectionName(bool AddSegment)
Return the name of data section containing names of instrumented functions.
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
void set(SummaryFieldKind K, uint64_t V)
InstrProfError(instrprof_error Err)
Function object to check whether the first component of a std::pair compares less than the first comp...
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
This is a value type class that represents a single section in the list of sections in the object fil...