15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
44#include <system_error>
52struct InstrProfRecord;
59#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
65 return std::numeric_limits<uint64_t>::max() - 2;
75 bool AddSegmentInfo =
true);
80 return INSTR_PROF_VALUE_PROF_FUNC_STR;
86 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
110 return "__llvm_prf_nm";
117 return "__llvm_coverage_mapping";
125 return "__llvm_coverage_names";
134 return "__llvm_profile_register_functions";
140 return "__llvm_profile_register_function";
145 return "__llvm_profile_register_names_function";
158 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
164 return "__llvm_profile_runtime_user";
168 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
178 uint64_t Version = INSTR_PROF_INDEX_VERSION);
187 uint64_t Version = INSTR_PROF_INDEX_VERSION);
215 StringRef PGOFuncName);
223 StringRef FileName =
"<unknown>");
235 bool doCompression, std::string &Result);
243 std::string &Result,
bool doCompression =
true);
255#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
263 const InstrProfRecord &InstrProfR,
269 ArrayRef<InstrProfValueData> VDs,
uint64_t Sum,
278 InstrProfValueData ValueData[],
280 bool GetNoICPValue =
false);
361 static std::vector<BPFunctionNode>
372 : Err(Err), Msg(ErrStr.str()) {
376 std::string
message()
const override;
390 static std::pair<instrprof_error, std::string>
take(
Error E) {
392 std::string Msg =
"";
414namespace IndexedInstrProf {
435 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
438 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
445 return "** External Symbol **";
454 inline void finalizeSymtab();
484 template <
typename NameIterRange>
Error create(
const NameIterRange &IterRange);
489 if (FuncName.
empty())
491 "function name is empty");
492 auto Ins = NameTab.
insert(FuncName);
494 MD5NameMap.push_back(std::make_pair(
504 AddrToMD5Map.push_back(std::make_pair(
Addr, MD5Val));
526 return Symbol == InstrProfSymtab::getExternalSymbol();
545template <
typename NameIterRange>
547 for (
auto Name : IterRange)
555void InstrProfSymtab::finalizeSymtab() {
561 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
569 return InstrProfSymtab::getExternalSymbol();
576 [](
const std::pair<uint64_t, StringRef> &
LHS,
578 if (Result != MD5NameMap.end() && Result->first ==
MD5Hash)
579 return Result->second;
586 [](
const std::pair<uint64_t, Function *> &
LHS,
588 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
589 return Result->second;
644 if (Sum1 < 1.0f || Sum2 < 1.0f)
646 return std::min(Val1 / Sum1, Val2 / Sum2);
662 template <
class InputIterator>
669 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
670 return left.Value < right.Value;
701 ValueData(
RHS.ValueData
702 ?
std::make_unique<ValueProfData>(*
RHS.ValueData)
708 if (!
RHS.ValueData) {
713 ValueData = std::make_unique<ValueProfData>(*
RHS.ValueData);
715 *ValueData = *
RHS.ValueData;
736 inline std::unique_ptr<InstrProfValueData[]>
765 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
766 for (
auto &SR : getValueSitesForKind(Kind))
816 struct ValueProfData {
817 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
818 std::vector<InstrProfValueSiteRecord> MemOPSizes;
820 std::unique_ptr<ValueProfData> ValueData;
822 MutableArrayRef<InstrProfValueSiteRecord>
823 getValueSitesForKind(
uint32_t ValueKind) {
827 auto AR =
const_cast<const InstrProfRecord *
>(
this)->getValueSitesForKind(
829 return MutableArrayRef(
830 const_cast<InstrProfValueSiteRecord *
>(AR.data()), AR.size());
832 ArrayRef<InstrProfValueSiteRecord>
833 getValueSitesForKind(
uint32_t ValueKind)
const {
837 case IPVK_IndirectCallTarget:
838 return ValueData->IndirectCallSites;
840 return ValueData->MemOPSizes;
846 std::vector<InstrProfValueSiteRecord> &
847 getOrCreateValueSitesForKind(
uint32_t ValueKind) {
849 ValueData = std::make_unique<ValueProfData>();
851 case IPVK_IndirectCallTarget:
852 return ValueData->IndirectCallSites;
854 return ValueData->MemOPSizes;
862 InstrProfSymtab *SymTab);
884 std::vector<uint64_t>
Counts)
887 std::vector<uint64_t>
Counts,
902 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
903 NumValueKinds += !(getValueSitesForKind(Kind).empty());
904 return NumValueKinds;
909 for (
const auto &SR : getValueSitesForKind(ValueKind))
910 N += SR.ValueData.size();
915 return getValueSitesForKind(ValueKind).size();
920 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
923std::unique_ptr<InstrProfValueData[]>
927 uint64_t &TotalCount = (TotalC ==
nullptr ? Dummy : *TotalC);
931 return std::unique_ptr<InstrProfValueData[]>(
nullptr);
934 auto VD = std::make_unique<InstrProfValueData[]>(
N);
945 for (
auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
946 Dest[
I].Value = V.Value;
947 Dest[
I].Count = V.Count;
957 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
961#define INSTR_PROF_VALUE_PROF_DATA
966 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
967 return left.Count > right.Count;
970 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
975namespace IndexedInstrProf {
1053 size_t size()
const;
1099 void operator delete(
void *ptr) { ::operator
delete(ptr); }
1107 return reinterpret_cast<const uint64_t *
>(
this + 1);
1111 return reinterpret_cast<uint64_t *
>(
this + 1);
1115 return reinterpret_cast<const Entry *
>(
1142 return std::unique_ptr<Summary>(
new (::operator
new(TotalSize))
1148namespace RawInstrProf {
1167 return INSTR_PROF_RAW_MAGIC_64;
1171 return INSTR_PROF_RAW_MAGIC_32;
1180 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1189#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Machine Check Debug Module
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StringSet - A set-like wrapper for the StringMap.
Base class for user error types.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
LinkageTypes
An enumeration for the kinds of linkage for global values.
void log(raw_ostream &OS) const override
Print an error message to an output stream.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
const std::string & getMessage() const
instrprof_error get() const
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
std::string message() const override
Return the error message as a string.
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
StringRef getFuncOrVarName(uint64_t ValMD5Hash)
Return name of functions or global variables from the name's md5 hash value.
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
StringRef getNameData() const
Return the name section data.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
InstrProfSymtab()=default
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash)
Just like getFuncOrVarName, except that it will return literal string 'External Symbol' if the functi...
A Module instance is used to store all the information related to an LLVM module.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
StringSet - A wrapper for StringMap that provides set-like functionality.
std::pair< typename Base::iterator, bool > insert(StringRef key)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
An efficient, type-erasing, non-owning reference to a callable.
This is a value type class that represents a single section in the list of sections in the object fil...
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
uint64_t getMagic< uint32_t >()
uint64_t getMagic< uint64_t >()
constexpr size_t NameSize
uint64_t MD5Hash(const FunctionId &Obj)
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
std::error_code make_error_code(BitcodeError E)
StringRef getPGOFuncNameMetadataName()
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
StringRef getInstrProfCounterBiasVarName()
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
std::pair< StringRef, StringRef > getParsedIRPGOFuncName(StringRef IRPGOFuncName)
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void sort(IteratorTy Start, IteratorTy End)
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
@ LLVM_MARK_AS_BITMASK_ENUM
@ unable_to_correlate_profile
@ raw_profile_version_mismatch
@ missing_debug_info_for_correlation
@ counter_value_too_large
@ value_site_count_mismatch
@ unexpected_debug_info_for_correlation
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
const std::error_category & instrprof_category()
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
InstrProfKind
An enum describing the attributes of an instrumented profile.
@ FunctionEntryInstrumentation
@ FrontendInstrumentation
Implement std::hash so that hash_code can be used in STL containers.
double ValueCounts[IPVK_Last - IPVK_First+1]
uint64_t Cutoff
The required percentile of total execution count.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
const Entry * getCutoffEntryBase() const
uint64_t get(SummaryFieldKind K) const
void set(SummaryFieldKind K, uint64_t V)
Entry * getCutoffEntryBase()
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
uint64_t * getSummaryDataBase()
@ TotalNumFunctions
The total number of functions instrumented.
@ TotalNumBlocks
Total number of instrumented blocks/edges.
@ MaxFunctionCount
The maximal execution count among all functions.
@ TotalBlockCount
The sum of all instrumented block counts.
@ MaxBlockCount
Max block count of the program.
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
const uint64_t * getSummaryDataBase() const
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumSummaryFields
const Entry & getEntry(uint32_t I) const
uint64_t NumCutoffEntries
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
InstrProfRecord()=default
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
InstrProfRecord(std::vector< uint64_t > Counts)
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
void setPseudoCount(CountPseudoKind Kind)
InstrProfRecord(InstrProfRecord &&)=default
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
InstrProfRecord & operator=(const InstrProfRecord &RHS)
void clearValueData()
Clear value data entries.
InstrProfRecord(const InstrProfRecord &RHS)
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
InstrProfRecord(std::vector< uint64_t > Counts, std::vector< uint8_t > BitmapBytes)
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void sortValueData()
Sort value profile data (per site) by count.
std::vector< uint8_t > BitmapBytes
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
InstrProfRecord & operator=(InstrProfRecord &&)=default
void Clear()
Clear value data entries and edge counters.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
void sortByCount()
Sort ValueData Descending by Count.
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
InstrProfValueSiteRecord()
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
static bool hasCSFlagInHash(uint64_t FuncHash)
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts, std::vector< uint8_t > BitmapBytes)
static void setCSFlagInHash(uint64_t &FuncHash)
static const int CS_FLAG_IN_FUNC_HASH
NamedInstrProfRecord()=default
const std::string NameFilter
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
void setFuncInfo(StringRef Name, uint64_t Hash)
CountSumOrPercent Mismatch
OverlapStats(OverlapStatsLevel L=ProgramLevel)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...
std::vector< uint64_t > FunctionNameRefs
TemporalProfTraceTy(std::initializer_list< uint64_t > Trace={}, uint64_t Weight=1)
Function object to check whether the first component of a container supported by std::get (like std::...