20#include "llvm/Config/config.h"
53#include <system_error>
62 cl::desc(
"Use full module build paths in the profile counter names for "
63 "static functions."));
75 cl::desc(
"Strip specified level of directory name from source path in "
76 "the profile counter name for static functions."));
79 const std::string &ErrMsg =
"") {
84 case instrprof_error::success:
87 case instrprof_error::eof:
90 case instrprof_error::unrecognized_format:
91 OS <<
"unrecognized instrumentation profile encoding format";
93 case instrprof_error::bad_magic:
94 OS <<
"invalid instrumentation profile data (bad magic)";
96 case instrprof_error::bad_header:
97 OS <<
"invalid instrumentation profile data (file header is corrupt)";
99 case instrprof_error::unsupported_version:
100 OS <<
"unsupported instrumentation profile format version";
102 case instrprof_error::unsupported_hash_type:
103 OS <<
"unsupported instrumentation profile hash type";
105 case instrprof_error::too_large:
106 OS <<
"too much profile data";
108 case instrprof_error::truncated:
109 OS <<
"truncated profile data";
111 case instrprof_error::malformed:
112 OS <<
"malformed instrumentation profile data";
114 case instrprof_error::missing_debug_info_for_correlation:
115 OS <<
"debug info for correlation is required";
117 case instrprof_error::unexpected_debug_info_for_correlation:
118 OS <<
"debug info for correlation is not necessary";
120 case instrprof_error::unable_to_correlate_profile:
121 OS <<
"unable to correlate profile";
123 case instrprof_error::invalid_prof:
124 OS <<
"invalid profile created. Please file a bug "
125 "at: " BUG_REPORT_URL
126 " and include the profraw files that caused this error.";
128 case instrprof_error::unknown_function:
129 OS <<
"no profile data available for function";
131 case instrprof_error::hash_mismatch:
132 OS <<
"function control flow change detected (hash mismatch)";
134 case instrprof_error::count_mismatch:
135 OS <<
"function basic block count change detected (counter mismatch)";
137 case instrprof_error::counter_overflow:
138 OS <<
"counter overflow";
140 case instrprof_error::value_site_count_mismatch:
141 OS <<
"function value site count change detected (counter mismatch)";
143 case instrprof_error::compress_failed:
144 OS <<
"failed to compress data (zlib)";
146 case instrprof_error::uncompress_failed:
147 OS <<
"failed to uncompress data (zlib)";
149 case instrprof_error::empty_raw_profile:
150 OS <<
"empty raw profile file";
152 case instrprof_error::zlib_unavailable:
153 OS <<
"profile uses zlib compression but the profile reader was built "
154 "without zlib support";
160 OS <<
": " << ErrMsg;
170class InstrProfErrorCategoryType :
public std::error_category {
171 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
173 std::string message(
int IE)
const override {
181 static InstrProfErrorCategoryType ErrorCategory;
182 return ErrorCategory;
187const char *InstrProfSectNameCommon[] = {
188#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
193const char *InstrProfSectNameCoff[] = {
194#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
199const char *InstrProfSectNamePrefix[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
210 "enable-name-compression",
215 bool AddSegmentInfo) {
216 std::string SectName;
219 SectName = InstrProfSectNamePrefix[IPSK];
222 SectName += InstrProfSectNameCoff[IPSK];
224 SectName += InstrProfSectNameCommon[IPSK];
226 if (OF ==
Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
227 SectName +=
",regular,live_support";
244 ++NumCountMismatches;
247 ++NumCounterOverflows;
250 ++NumValueSiteCountMismatches;
276 for (
auto & CI : PathNameStr) {
285 return PathNameStr.
substr(LastPos);
303 StringRef FileName(
F.getParent()->getSourceFileName());
314 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
325 if (FileName.
empty())
344 const char *InvalidChars =
"-:<>/\"'";
345 size_t found = VarName.find_first_of(InvalidChars);
346 while (found != std::string::npos) {
347 VarName[found] =
'_';
348 found = VarName.find_first_of(InvalidChars, found + 1);
404 const std::string UniqSuffix =
".__uniq.";
405 auto pos = PGOFuncName.find(UniqSuffix);
408 if (pos != std::string::npos)
409 pos += UniqSuffix.length();
412 pos = PGOFuncName.find(
'.', pos);
413 if (pos != std::string::npos && pos != 0) {
414 const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
427 auto It =
partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t>
A) {
428 return A.first < Address;
434 if (It != AddrToMD5Map.end() && It->first == Address)
440 bool doCompression, std::string &Result) {
441 assert(!NameStrs.
empty() &&
"No name data to emit");
443 uint8_t Header[16], *
P = Header;
444 std::string UncompressedNameStrings =
449 "PGO name is invalid (contains separator token)");
451 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
454 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
457 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
458 unsigned HeaderLen =
P - &Header[0];
459 Result.append(HeaderStr, HeaderLen);
464 if (!doCompression) {
465 return WriteStringToResult(0, UncompressedNameStrings);
470 CompressedNameStrings,
473 return WriteStringToResult(CompressedNameStrings.
size(),
480 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
485 std::string &Result,
bool doCompression) {
486 std::vector<std::string> NameStrs;
487 for (
auto *NameVar : NameVars) {
496 const uint8_t *EndP = NameStrings.
bytes_end();
503 bool isCompressed = (CompressedSize != 0);
511 UncompressedNameStrings,
517 NameStrings =
toStringRef(UncompressedNameStrings);
520 StringRef(
reinterpret_cast<const char *
>(
P), UncompressedSize);
521 P += UncompressedSize;
530 while (
P < EndP && *
P == 0)
543 for (
uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
546 for (
size_t I = 0;
I < NumValueSites; ++
I) {
550 KindSum += VD[V].Count;
562 double Score = 0.0f, FuncLevelScore = 0.0f;
567 while (
I != IE && J != JE) {
568 if (
I->Value == J->Value) {
576 }
else if (
I->Value < J->Value) {
592 assert(ThisNumValueSites ==
Other.getNumValueSites(ValueKind));
593 if (!ThisNumValueSites)
596 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
597 getOrCreateValueSitesForKind(ValueKind);
599 Other.getValueSitesForKind(ValueKind);
601 ThisSiteRecords[
I].
overlap(OtherSiteRecords[
I], ValueKind, Overlap,
611 bool Mismatch = (
Counts.size() !=
Other.Counts.size());
615 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
618 if (ThisNumValueSites != OtherNumValueSites) {
630 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
636 for (
size_t I = 0,
E =
Other.Counts.size();
I <
E; ++
I) {
639 MaxCount = std::max(
Other.Counts[
I], MaxCount);
644 if (MaxCount >= ValueCutoff) {
645 double FuncScore = 0.0;
646 for (
size_t I = 0,
E =
Other.Counts.size();
I <
E; ++
I)
652 FuncLevelOverlap.
Valid =
true;
663 for (
const InstrProfValueData &J : Input.
ValueData) {
664 while (
I != IE &&
I->Value < J.Value)
666 if (
I != IE &&
I->Value == J.Value) {
690void InstrProfRecord::mergeValueProfData(
694 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
695 if (ThisNumValueSites != OtherNumValueSites) {
699 if (!ThisNumValueSites)
701 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
702 getOrCreateValueSitesForKind(ValueKind);
704 Src.getValueSitesForKind(ValueKind);
706 ThisSiteRecords[
I].
merge(OtherSiteRecords[
I], Weight, Warn);
736 for (
size_t I = 0,
E =
Other.Counts.size();
I <
E; ++
I) {
749 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
750 mergeValueProfData(Kind,
Other, Weight, Warn);
753void InstrProfRecord::scaleValueProfData(
756 for (
auto &R : getValueSitesForKind(ValueKind))
762 assert(
D != 0 &&
"D cannot be 0");
763 for (
auto &Count : this->
Counts) {
773 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
774 scaleValueProfData(Kind,
N,
D, Warn);
783 if (ValueKind == IPVK_IndirectCallTarget)
795 std::vector<InstrProfValueSiteRecord> &ValueSites =
796 getOrCreateValueSitesForKind(ValueKind);
798 ValueSites.emplace_back();
800 ValueSites.emplace_back(VData, VData +
N);
803#define INSTR_PROF_COMMON_API_IMPL
817 ->getNumValueSites(VKind);
822 ->getNumValueData(VKind);
828 ->getNumValueDataForSite(VK, S);
833 reinterpret_cast<const InstrProfRecord *
>(R)->getValueForSite(Dst, K, S);
838 (ValueProfData *)(
new (::operator
new(TotalSizeInBytes)) ValueProfData());
839 memset(VD, 0, TotalSizeInBytes);
857 return getValueProfDataSize(&Closure);
861std::unique_ptr<ValueProfData>
865 std::unique_ptr<ValueProfData> VPD(
872 Record.reserveSites(Kind, NumValueSites);
874 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
875 for (
uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
876 uint8_t ValueDataCount = this->SiteCountArray[VSite];
877 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
878 ValueData += ValueDataCount;
887 using namespace support;
893 sys::swapByteOrder<uint32_t>(NumValueSites);
894 sys::swapByteOrder<uint32_t>(Kind);
896 uint32_t ND = getValueProfRecordNumValueData(
this);
897 InstrProfValueData *VD = getValueProfRecordValueData(
this);
901 sys::swapByteOrder<uint64_t>(VD[
I].
Value);
902 sys::swapByteOrder<uint64_t>(VD[
I].Count);
905 sys::swapByteOrder<uint32_t>(NumValueSites);
906 sys::swapByteOrder<uint32_t>(Kind);
912 if (NumValueKinds == 0)
915 ValueProfRecord *VR = getFirstValueProfRecord(
this);
916 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
917 VR->deserializeTo(
Record, SymTab);
918 VR = getValueProfRecordNext(VR);
924 using namespace support;
927 return endian::readNext<T, little, unaligned>(
D);
929 return endian::readNext<T, big, unaligned>(
D);
933 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
937Error ValueProfData::checkIntegrity() {
938 if (NumValueKinds > IPVK_Last + 1)
939 return make_error<InstrProfError>(
943 return make_error<InstrProfError>(
946 ValueProfRecord *VR = getFirstValueProfRecord(
this);
947 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
948 if (VR->Kind > IPVK_Last)
950 "value kind is invalid");
951 VR = getValueProfRecordNext(VR);
952 if ((
char *)VR - (
char *)
this > (
ptrdiff_t)TotalSize)
953 return make_error<InstrProfError>(
955 "value profile address is greater than total size");
961ValueProfData::getValueProfData(
const unsigned char *
D,
962 const unsigned char *
const BufferEnd,
964 using namespace support;
966 if (
D +
sizeof(ValueProfData) > BufferEnd)
969 const unsigned char *Header =
D;
970 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
971 if (
D + TotalSize > BufferEnd)
975 memcpy(VPD.get(),
D, TotalSize);
977 VPD->swapBytesToHost(Endianness);
979 Error E = VPD->checkIntegrity();
983 return std::move(VPD);
987 using namespace support;
992 sys::swapByteOrder<uint32_t>(TotalSize);
993 sys::swapByteOrder<uint32_t>(NumValueKinds);
995 ValueProfRecord *VR = getFirstValueProfRecord(
this);
996 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
998 VR = getValueProfRecordNext(VR);
1003 using namespace support;
1008 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1009 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1010 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1014 sys::swapByteOrder<uint32_t>(TotalSize);
1015 sys::swapByteOrder<uint32_t>(NumValueKinds);
1027 std::unique_ptr<InstrProfValueData[]> VD =
1052 for (
auto &VD : VDs) {
1066 InstrProfValueData ValueData[],
1068 bool GetNoICPValue) {
1083 if (!
Tag->getString().equals(
"VP"))
1099 ActualNumValueData = 0;
1101 for (
unsigned I = 3;
I < NOps;
I += 2) {
1102 if (ActualNumValueData >= MaxNumValueData)
1106 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1107 if (!
Value || !Count)
1112 ValueData[ActualNumValueData].Value =
Value->getZExtValue();
1113 ValueData[ActualNumValueData].Count = CntValue;
1114 ActualNumValueData++;
1125 if (PGOFuncName ==
F.getName())
1163 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1164 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1169 if (IRInstrVar->isDeclaration())
1173 if (!IRInstrVar->hasInitializer())
1176 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1179 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1184 if (
F.getName().empty())
1190 if (CheckAddressTaken &&
F.hasAddressTaken())
1198 if (!
F.hasComdat()) {
1207 if (InstrProfileOutput.
empty())
1213 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1215 Triple TT(M.getTargetTriple());
1216 if (TT.supportsCOMDAT()) {
1218 ProfileNameVar->
setComdat(M.getOrInsertComdat(
1219 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1224 const std::string &TestFilename,
1226 auto getProfileSum = [IsCS](
const std::string &Filename,
1232 if (
Error E = ReaderOrErr.takeError()) {
1235 auto Reader = std::move(ReaderOrErr.get());
1236 Reader->accumulateCounts(Sum, IsCS);
1254 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1264 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1274 const char *EntryName =
1277 OS <<
"Profile overlap infomation for base_profile: " << *
BaseFilename
1278 <<
" and test_profile: " << *
TestFilename <<
"\nProgram level:\n";
1280 OS <<
"Function level:\n"
1289 OS <<
" # of " << EntryName
1295 OS <<
" Mismatched count percentage (Edge): "
1298 OS <<
" Percentage of Edge profile only in test_profile: "
1305 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1308 char ProfileKindName[20];
1310 case IPVK_IndirectCallTarget:
1311 strncpy(ProfileKindName,
"IndirectCall", 19);
1313 case IPVK_MemOPSize:
1314 strncpy(ProfileKindName,
"MemOP", 19);
1317 snprintf(ProfileKindName, 19,
"VP[%d]",
I);
1320 OS <<
" " << ProfileKindName
1324 OS <<
" Mismatched count percentage (" << ProfileKindName
1327 OS <<
" Percentage of " << ProfileKindName
1328 <<
" profile only in test_profile: "
1330 OS <<
" " << ProfileKindName
1333 <<
" " << ProfileKindName
1339namespace IndexedInstrProf {
1341template <
typename T1,
typename T2>
1352 using namespace support;
1353 return endian::byte_swap<uint64_t, little>(
Version);
1357 using namespace support;
1358 static_assert(std::is_standard_layout_v<Header>,
1359 "The header should be standard layout type since we use offset "
1360 "of fields to read.");
1371 if (GET_VERSION(
H.formatVersion()) >
1375 switch (GET_VERSION(
H.formatVersion())) {
1380 "Please update the reading code below if a new field has been added, "
1381 "if not add a case statement to fall through to the latest version.");
1402 "Please update the size computation below if a new field has "
1403 "been added to the header, if not add a case statement to "
1404 "fall through to the latest version.");
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallString class.
This file defines the SmallVector class.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
This is the shared class of boolean and integer constants.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
void setComdat(Comdat *C)
static bool isLocalLinkage(LinkageTypes Linkage)
void setLinkage(LinkageTypes LT)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
bool isDiscardableIfUnused() const
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ ExternalWeakLinkage
ExternalWeak linkage description.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
std::string message() const override
Return the error message as a string.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
This is an important class for using LLVM in a threaded context.
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
MDString * createString(StringRef Str)
Return the given string as metadata.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string str() const
str - Get the contents as an std::string.
const unsigned char * bytes_end() const
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
bool startswith(StringRef Prefix) const
const unsigned char * bytes_begin() const
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
static uint64_t read(const unsigned char *Buffer, size_t Offset)
size_t constexpr offsetOf(T1 T2::*Member)
initializer< Ty > init(const Ty &Val)
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
StringRef getPGOFuncNameMetadataName()
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig)
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
support::endianness getHostEndianness()
@ value_site_count_mismatch
const std::error_category & instrprof_category()
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
static ValueProfRecordClosure InstrProfRecordClosure
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
void consumeError(Error Err)
Consume a Error without doing anything.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
double ValueCounts[IPVK_Last - IPVK_First+1]
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
void setPseudoCount(CountPseudoKind Kind)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
CountSumOrPercent Mismatch