15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
44#include <system_error>
52struct InstrProfRecord;
59#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
65 return std::numeric_limits<uint64_t>::max() - 2;
75 bool AddSegmentInfo =
true);
80 return INSTR_PROF_VALUE_PROF_FUNC_STR;
86 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
107 return "__llvm_prf_nm";
114 return "__llvm_coverage_mapping";
122 return "__llvm_coverage_names";
131 return "__llvm_profile_register_functions";
137 return "__llvm_profile_register_function";
142 return "__llvm_profile_register_names_function";
155 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
161 return "__llvm_profile_runtime_user";
165 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
175 uint64_t Version = INSTR_PROF_INDEX_VERSION);
184 uint64_t Version = INSTR_PROF_INDEX_VERSION);
203 StringRef PGOFuncName);
211 StringRef FileName =
"<unknown>");
223 bool doCompression, std::string &Result);
228 std::string &Result,
bool doCompression =
true);
245#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
253 const InstrProfRecord &InstrProfR,
259 ArrayRef<InstrProfValueData> VDs,
uint64_t Sum,
268 InstrProfValueData ValueData[],
270 bool GetNoICPValue =
false);
351 : Err(Err), Msg(ErrStr.str()) {
355 std::string
message()
const override;
369 static std::pair<instrprof_error, std::string>
take(
Error E) {
371 std::string Msg =
"";
395 unsigned NumHashMismatches = 0;
398 unsigned NumCountMismatches = 0;
401 unsigned NumCounterOverflows = 0;
404 unsigned NumValueSiteCountMismatches = 0;
411 "Unchecked soft error encountered");
428 return NumValueSiteCountMismatches;
436 auto E = make_error<InstrProfError>(FirstError);
448namespace IndexedInstrProf {
469 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
472 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
479 return "** External Symbol **";
486 inline void finalizeSymtab();
516 template <
typename NameIterRange>
Error create(
const NameIterRange &IterRange);
521 if (FuncName.
empty())
523 "function name is empty");
524 auto Ins = NameTab.
insert(FuncName);
526 MD5NameMap.push_back(std::make_pair(
536 AddrToMD5Map.push_back(std::make_pair(
Addr, MD5Val));
558 return Symbol == InstrProfSymtab::getExternalSymbol();
589template <
typename NameIterRange>
591 for (
auto Name : IterRange)
599void InstrProfSymtab::finalizeSymtab() {
605 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
613 return InstrProfSymtab::getExternalSymbol();
620 [](
const std::pair<uint64_t, StringRef> &
LHS,
622 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
623 return Result->second;
630 [](
const std::pair<uint64_t, Function *> &
LHS,
632 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
633 return Result->second;
698 if (Sum1 < 1.0f || Sum2 < 1.0f)
700 return std::min(Val1 / Sum1, Val2 / Sum2);
716 template <
class InputIterator>
723 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
724 return left.Value < right.Value;
751 ValueData(
RHS.ValueData
752 ?
std::make_unique<ValueProfData>(*
RHS.ValueData)
757 if (!
RHS.ValueData) {
762 ValueData = std::make_unique<ValueProfData>(*
RHS.ValueData);
764 *ValueData = *
RHS.ValueData;
785 inline std::unique_ptr<InstrProfValueData[]>
814 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
815 for (
auto &SR : getValueSitesForKind(Kind))
865 struct ValueProfData {
866 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
867 std::vector<InstrProfValueSiteRecord> MemOPSizes;
869 std::unique_ptr<ValueProfData> ValueData;
871 MutableArrayRef<InstrProfValueSiteRecord>
872 getValueSitesForKind(
uint32_t ValueKind) {
876 auto AR =
const_cast<const InstrProfRecord *
>(
this)->getValueSitesForKind(
878 return MutableArrayRef(
879 const_cast<InstrProfValueSiteRecord *
>(AR.data()), AR.size());
881 ArrayRef<InstrProfValueSiteRecord>
882 getValueSitesForKind(
uint32_t ValueKind)
const {
886 case IPVK_IndirectCallTarget:
887 return ValueData->IndirectCallSites;
889 return ValueData->MemOPSizes;
895 std::vector<InstrProfValueSiteRecord> &
896 getOrCreateValueSitesForKind(
uint32_t ValueKind) {
898 ValueData = std::make_unique<ValueProfData>();
900 case IPVK_IndirectCallTarget:
901 return ValueData->IndirectCallSites;
903 return ValueData->MemOPSizes;
911 InstrProfSymtab *SymTab);
933 std::vector<uint64_t>
Counts)
946 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
947 NumValueKinds += !(getValueSitesForKind(Kind).empty());
948 return NumValueKinds;
953 for (
const auto &SR : getValueSitesForKind(ValueKind))
954 N += SR.ValueData.size();
959 return getValueSitesForKind(ValueKind).size();
964 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
967std::unique_ptr<InstrProfValueData[]>
971 uint64_t &TotalCount = (TotalC ==
nullptr ? Dummy : *TotalC);
975 return std::unique_ptr<InstrProfValueData[]>(
nullptr);
978 auto VD = std::make_unique<InstrProfValueData[]>(
N);
989 for (
auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
990 Dest[
I].Value = V.Value;
991 Dest[
I].Count = V.Count;
1001 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
1009#define INSTR_PROF_VALUE_PROF_DATA
1014 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
1015 return left.Count > right.Count;
1018 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
1023namespace IndexedInstrProf {
1099 size_t size()
const;
1145 void operator delete(
void *ptr) { ::operator
delete(ptr); }
1153 return reinterpret_cast<const uint64_t *
>(
this + 1);
1157 return reinterpret_cast<uint64_t *
>(
this + 1);
1161 return reinterpret_cast<const Entry *
>(
1188 return std::unique_ptr<Summary>(
new (::operator
new(TotalSize))
1194namespace RawInstrProf {
1212 return INSTR_PROF_RAW_MAGIC_64;
1216 return INSTR_PROF_RAW_MAGIC_32;
1225 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1234#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1242 int64_t &RangeLast);
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Machine Check Debug Module
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StringSet - A set-like wrapper for the StringMap.
Base class for user error types.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
LinkageTypes
An enumeration for the kinds of linkage for global values.
void log(raw_ostream &OS) const override
Print an error message to an output stream.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
const std::string & getMessage() const
instrprof_error get() const
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
std::string message() const override
Return the error message as a string.
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
StringRef getOrigFuncName(uint64_t FuncMD5Hash)
Return the function's original assembly name by stripping off the prefix attached (to symbols with pr...
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
StringRef getNameData() const
Return the name section data.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
InstrProfSymtab()=default
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
A Module instance is used to store all the information related to an LLVM module.
Error takeError()
Return the first encountered error and reset FirstError to a success value.
unsigned getNumHashMismatches() const
Get the number of hash mismatches.
SoftInstrProfErrors()=default
unsigned getNumCountMismatches() const
Get the number of count mismatches.
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
unsigned getNumValueSiteCountMismatches() const
Get the number of value site count mismatches.
unsigned getNumCounterOverflows() const
Get the number of counter overflows.
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
static constexpr size_t npos
StringSet - A wrapper for StringMap that provides set-like functionality.
std::pair< typename Base::iterator, bool > insert(StringRef key)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
An efficient, type-erasing, non-owning reference to a callable.
This is a value type class that represents a single section in the list of sections in the object fil...
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
uint64_t getMagic< uint32_t >()
uint64_t getMagic< uint64_t >()
constexpr size_t NameSize
static const bool IsLittleEndianHost
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::error_code make_error_code(BitcodeError E)
StringRef getPGOFuncNameMetadataName()
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
StringRef getInstrProfCounterBiasVarName()
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
void sort(IteratorTy Start, IteratorTy End)
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
support::endianness getHostEndianness()
@ LLVM_MARK_AS_BITMASK_ENUM
@ unable_to_correlate_profile
@ raw_profile_version_mismatch
@ missing_debug_info_for_correlation
@ value_site_count_mismatch
@ unexpected_debug_info_for_correlation
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
const std::error_category & instrprof_category()
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast)
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
InstrProfKind
An enum describing the attributes of an instrumented profile.
@ FunctionEntryInstrumentation
@ FrontendInstrumentation
double ValueCounts[IPVK_Last - IPVK_First+1]
uint64_t Cutoff
The required percentile of total execution count.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
const Entry * getCutoffEntryBase() const
uint64_t get(SummaryFieldKind K) const
void set(SummaryFieldKind K, uint64_t V)
Entry * getCutoffEntryBase()
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
uint64_t * getSummaryDataBase()
@ TotalNumFunctions
The total number of functions instrumented.
@ TotalNumBlocks
Total number of instrumented blocks/edges.
@ MaxFunctionCount
The maximal execution count among all functions.
@ TotalBlockCount
The sum of all instrumented block counts.
@ MaxBlockCount
Max block count of the program.
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
const uint64_t * getSummaryDataBase() const
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumSummaryFields
const Entry & getEntry(uint32_t I) const
uint64_t NumCutoffEntries
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
InstrProfRecord()=default
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
InstrProfRecord(std::vector< uint64_t > Counts)
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
void setPseudoCount(CountPseudoKind Kind)
InstrProfRecord(InstrProfRecord &&)=default
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
InstrProfRecord & operator=(const InstrProfRecord &RHS)
void clearValueData()
Clear value data entries.
InstrProfRecord(const InstrProfRecord &RHS)
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void sortValueData()
Sort value profile data (per site) by count.
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
InstrProfRecord & operator=(InstrProfRecord &&)=default
void Clear()
Clear value data entries and edge counters.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
void sortByCount()
Sort ValueData Descending by Count.
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
InstrProfValueSiteRecord()
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
static bool hasCSFlagInHash(uint64_t FuncHash)
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
static void setCSFlagInHash(uint64_t &FuncHash)
static const int CS_FLAG_IN_FUNC_HASH
NamedInstrProfRecord()=default
const std::string NameFilter
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
void setFuncInfo(StringRef Name, uint64_t Hash)
CountSumOrPercent Mismatch
OverlapStats(OverlapStatsLevel L=ProgramLevel)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
std::vector< uint64_t > FunctionNameRefs
Function object to check whether the first component of a container supported by std::get (like std::...