Go to the documentation of this file.
15 #ifndef LLVM_PROFILEDATA_INSTRPROF_H
16 #define LLVM_PROFILEDATA_INSTRPROF_H
44 #include <system_error>
52 struct InstrProfRecord;
53 class InstrProfSymtab;
59 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
75 bool AddSegmentInfo =
true);
80 return INSTR_PROF_VALUE_PROF_FUNC_STR;
86 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
107 return "__llvm_prf_nm";
114 return "__llvm_coverage_mapping";
122 return "__llvm_coverage_names";
131 return "__llvm_profile_register_functions";
137 return "__llvm_profile_register_function";
142 return "__llvm_profile_register_names_function";
155 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
161 return "__llvm_profile_runtime_user";
165 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
203 StringRef PGOFuncName);
211 StringRef FileName =
"<unknown>");
223 bool doCompression, std::string &Result);
228 std::string &Result,
bool doCompression =
true);
245 #define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
253 const InstrProfRecord &InstrProfR,
259 ArrayRef<InstrProfValueData> VDs,
uint64_t Sum,
268 InstrProfValueData ValueData[],
270 bool GetNoICPValue =
false);
341 : Err(Err), Msg(ErrStr.str()) {
345 std::string
message()
const override;
382 unsigned NumHashMismatches = 0;
385 unsigned NumCountMismatches = 0;
388 unsigned NumCounterOverflows = 0;
391 unsigned NumValueSiteCountMismatches = 0;
398 "Unchecked soft error encountered");
415 return NumValueSiteCountMismatches;
423 auto E = make_error<InstrProfError>(FirstError);
435 namespace IndexedInstrProf {
456 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
459 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
466 return "** External Symbol **";
473 inline void finalizeSymtab();
503 template <
typename NameIterRange>
Error create(
const NameIterRange &IterRange);
508 if (FuncName.
empty())
510 "function name is empty");
513 MD5NameMap.push_back(std::make_pair(
523 AddrToMD5Map.push_back(std::make_pair(
Addr, MD5Val));
545 return Symbol == InstrProfSymtab::getExternalSymbol();
576 template <
typename NameIterRange>
578 for (
auto Name : IterRange)
586 void InstrProfSymtab::finalizeSymtab() {
592 AddrToMD5Map.erase(
std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
600 return InstrProfSymtab::getExternalSymbol();
607 [](
const std::pair<uint64_t, StringRef> &
LHS,
609 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
610 return Result->second;
617 [](
const std::pair<uint64_t, Function *> &
LHS,
619 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
620 return Result->second;
685 if (Sum1 < 1.0
f || Sum2 < 1.0
f)
687 return std::min(Val1 / Sum1, Val2 / Sum2);
703 template <
class InputIterator>
710 [](
const InstrProfValueData &left,
const InstrProfValueData &
right) {
711 return left.Value <
right.Value;
738 ValueData(
RHS.ValueData
739 ?
std::make_unique<ValueProfData>(*
RHS.ValueData)
744 if (!
RHS.ValueData) {
749 ValueData = std::make_unique<ValueProfData>(*
RHS.ValueData);
751 *ValueData = *
RHS.ValueData;
772 inline std::unique_ptr<InstrProfValueData[]>
801 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
802 for (
auto &SR : getValueSitesForKind(Kind))
852 struct ValueProfData {
853 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
854 std::vector<InstrProfValueSiteRecord> MemOPSizes;
856 std::unique_ptr<ValueProfData> ValueData;
858 MutableArrayRef<InstrProfValueSiteRecord>
863 auto AR =
const_cast<const InstrProfRecord *
>(
this)->getValueSitesForKind(
866 const_cast<InstrProfValueSiteRecord *
>(AR.data()), AR.size());
868 ArrayRef<InstrProfValueSiteRecord>
873 case IPVK_IndirectCallTarget:
874 return ValueData->IndirectCallSites;
876 return ValueData->MemOPSizes;
882 std::vector<InstrProfValueSiteRecord> &
885 ValueData = std::make_unique<ValueProfData>();
887 case IPVK_IndirectCallTarget:
888 return ValueData->IndirectCallSites;
890 return ValueData->MemOPSizes;
898 InstrProfSymtab *SymTab);
920 std::vector<uint64_t>
Counts)
933 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
934 NumValueKinds += !(getValueSitesForKind(Kind).empty());
935 return NumValueKinds;
940 for (
const auto &SR : getValueSitesForKind(
ValueKind))
941 N += SR.ValueData.size();
946 return getValueSitesForKind(
ValueKind).size();
951 return getValueSitesForKind(
ValueKind)[Site].ValueData.size();
954 std::unique_ptr<InstrProfValueData[]>
958 uint64_t &TotalCount = (TotalC ==
nullptr ?
Dummy : *TotalC);
962 return std::unique_ptr<InstrProfValueData[]>(
nullptr);
965 auto VD = std::make_unique<InstrProfValueData[]>(
N);
976 for (
auto V : getValueSitesForKind(
ValueKind)[Site].ValueData) {
977 Dest[
I].Value = V.Value;
978 Dest[
I].Count = V.Count;
988 getOrCreateValueSitesForKind(
ValueKind).reserve(NumValueSites);
996 #define INSTR_PROF_VALUE_PROF_DATA
1001 [](
const InstrProfValueData &left,
const InstrProfValueData &
right) {
1002 return left.Count >
right.Count;
1005 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
1010 namespace IndexedInstrProf {
1083 size_t size()
const;
1129 void operator delete(
void *ptr) { ::operator
delete(ptr); }
1137 return reinterpret_cast<const uint64_t *
>(
this + 1);
1141 return reinterpret_cast<uint64_t *
>(
this + 1);
1145 return reinterpret_cast<const Entry *
>(
1172 return std::unique_ptr<Summary>(
new (::
operator new(TotalSize))
1178 namespace RawInstrProf {
1196 return INSTR_PROF_RAW_MAGIC_64;
1200 return INSTR_PROF_RAW_MAGIC_32;
1209 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1218 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1226 int64_t &RangeLast);
1236 #endif // LLVM_PROFILEDATA_INSTRPROF_H
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast)
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
SoftInstrProfErrors()=default
the custom lowered code happens to be right
const Entry * getCutoffEntryBase() const
InstrProfRecord()=default
This is an optimization pass for GlobalISel generic memory operations.
@ FunctionEntryInstrumentation
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
InstrProfRecord & operator=(const InstrProfRecord &RHS)
std::string message() const override
Return the error message as a string.
CountPseudoKind getCountPseudoKind() const
#define LLVM_MARK_AS_BITMASK_ENUM(LargestValue)
LLVM_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you can perform bitwise operatio...
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
static constexpr size_t npos
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
uint64_t ComputeHash(StringRef K)
void setPseudoCount(CountPseudoKind Kind)
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void dump(raw_fd_ostream &OS) const
instrprof_error get() const
void Clear()
Clear value data entries and edge counters.
static ErrorSuccess success()
Create a success value.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
const Entry & getEntry(uint32_t I) const
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
InstrProfRecord & operator=(InstrProfRecord &&)=default
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
The instances of the Type class are immutable: once they are created, they are never changed.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void clearValueData()
Clear value data entries.
InstrProfValueSiteRecord()
unsigned getNumCountMismatches() const
Get the number of count mismatches.
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
@ value_site_count_mismatch
to esp esp setne al movzbw ax esp setg cl movzbw cx cmove cx cl jne LBB1_2 esp ret(also really horrible code on ppc). This is due to the expand code for 64-bit compares. GCC produces multiple branches
InstrProfSymtab()=default
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
std::pair< typename Base::iterator, bool > insert(StringRef key)
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
Tagged union holding either a T or a Error.
void sortByCount()
Sort ValueData Descending by Count.
LinkageTypes
An enumeration for the kinds of linkage for global values.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
Profiling information for a single function.
static const int CS_FLAG_IN_FUNC_HASH
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Entry * getCutoffEntryBase()
@ unexpected_debug_info_for_correlation
const std::string * TestFilename
uint64_t * getSummaryDataBase()
NamedInstrProfRecord()=default
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
uint64_t getMagic< uint32_t >()
Itanium Name Demangler i e convert the string _Z1fv into f()". You can also use the CRTP base ManglingParser to perform some simple analysis on the mangled name
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Function object to check whether the first component of a std::pair compares less than the first comp...
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
unsigned getNumCounterOverflows() const
Get the number of counter overflows.
const std::string & getMessage() const
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
uint64_t getMagic< uint64_t >()
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
This class implements an extremely fast bulk output stream that can only output to a stream.
Error takeError()
Return the first encountered error and reset FirstError to a success value.
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
@ TotalBlockCount
The sum of all instrumented block counts.
static instrprof_error take(Error E)
Consume an Error and return the raw enum value contained within it.
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
This is a value type class that represents a single section in the list of sections in the object fil...
void sort(IteratorTy Start, IteratorTy End)
An efficient, type-erasing, non-owning reference to a callable.
support::endianness getHostEndianness()
constexpr bool empty() const
empty - Check if the string is empty.
@ MaxFunctionCount
The maximal execution count among all functions.
uint64_t get(SummaryFieldKind K) const
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
void set(SummaryFieldKind K, uint64_t V)
bar al al movzbl eax ret Missed when stored in a memory object
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
@ FrontendInstrumentation
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
A Module instance is used to store all the information related to an LLVM module.
Base class for user error types.
@ TotalNumFunctions
The total number of functions instrumented.
StringRef getOrigFuncName(uint64_t FuncMD5Hash)
Return the function's original assembly name by stripping off the prefix attached (to symbols with pr...
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
OverlapStats(OverlapStatsLevel L=ProgramLevel)
unsigned getNumValueSiteCountMismatches() const
Get the number of value site count mismatches.
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
void log(raw_ostream &OS) const override
Print an error message to an output stream.
StringSet - A wrapper for StringMap that provides set-like functionality.
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
void addOneUnique(const CountSumOrPercent &UniqueFunc)
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
static void setCSFlagInHash(uint64_t &FuncHash)
StringRef - Represent a constant reference to a string, i.e.
uint64_t Cutoff
The required percentile of total execution count.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static bool hasCSFlagInHash(uint64_t FuncHash)
uint64_t NumCutoffEntries
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Machine Check Debug Module
Linkage
Describes symbol linkage.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
A raw_ostream that writes to a file descriptor.
const std::error_category & instrprof_category()
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
std::error_code make_error_code(BitcodeError E)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
cl::opt< bool > DoInstrProfNameCompression
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
iterator_range< StringMapKeyIterator< std::nullopt_t > > keys() const
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
StringRef getNameData() const
Return the name section data.
InstrProfRecord(std::vector< uint64_t > Counts)
Lightweight error class with error context and mandatory checking.
InstrProfRecord(const InstrProfRecord &RHS)
@ MaxBlockCount
Max block count of the program.
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
CountSumOrPercent Overlap
CountSumOrPercent Mismatch
StringRef getPGOFuncNameMetadataName()
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
void sortValueData()
Sort value profile data (per site) by count.
static const bool IsLittleEndianHost
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
unsigned getNumHashMismatches() const
Get the number of hash mismatches.
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
void setFuncInfo(StringRef Name, uint64_t Hash)
const std::string * BaseFilename
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
@ unable_to_correlate_profile
StringRef getInstrProfCounterBiasVarName()
const std::string NameFilter
InstrProfKind
An enum describing the attributes of an instrumented profile.
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
void sortByTargetValues()
Sort ValueData ascending by Value.
double ValueCounts[IPVK_Last - IPVK_First+1]
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
@ TotalNumBlocks
Total number of instrumented blocks/edges.
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
uint64_t NumSummaryFields
uint64_t MinBlockCount
The minimum execution count for this percentile.
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
std::vector< uint64_t > Counts
const uint64_t * getSummaryDataBase() const
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
@ missing_debug_info_for_correlation
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
InstrProfValueSiteRecord(InputIterator F, InputIterator L)