20#include "llvm/Config/config.h"
54#include <system_error>
61#define DEBUG_TYPE "instrprof"
65 cl::desc(
"Use full module build paths in the profile counter names for "
66 "static functions."));
78 cl::desc(
"Strip specified level of directory name from source path in "
79 "the profile counter name for static functions."));
82 const std::string &ErrMsg =
"") {
87 case instrprof_error::success:
90 case instrprof_error::eof:
93 case instrprof_error::unrecognized_format:
94 OS <<
"unrecognized instrumentation profile encoding format";
96 case instrprof_error::bad_magic:
97 OS <<
"invalid instrumentation profile data (bad magic)";
99 case instrprof_error::bad_header:
100 OS <<
"invalid instrumentation profile data (file header is corrupt)";
102 case instrprof_error::unsupported_version:
103 OS <<
"unsupported instrumentation profile format version";
105 case instrprof_error::unsupported_hash_type:
106 OS <<
"unsupported instrumentation profile hash type";
108 case instrprof_error::too_large:
109 OS <<
"too much profile data";
111 case instrprof_error::truncated:
112 OS <<
"truncated profile data";
114 case instrprof_error::malformed:
115 OS <<
"malformed instrumentation profile data";
117 case instrprof_error::missing_correlation_info:
118 OS <<
"debug info/binary for correlation is required";
120 case instrprof_error::unexpected_correlation_info:
121 OS <<
"debug info/binary for correlation is not necessary";
123 case instrprof_error::unable_to_correlate_profile:
124 OS <<
"unable to correlate profile";
126 case instrprof_error::invalid_prof:
127 OS <<
"invalid profile created. Please file a bug "
128 "at: " BUG_REPORT_URL
129 " and include the profraw files that caused this error.";
131 case instrprof_error::unknown_function:
132 OS <<
"no profile data available for function";
134 case instrprof_error::hash_mismatch:
135 OS <<
"function control flow change detected (hash mismatch)";
137 case instrprof_error::count_mismatch:
138 OS <<
"function basic block count change detected (counter mismatch)";
140 case instrprof_error::bitmap_mismatch:
141 OS <<
"function bitmap size change detected (bitmap size mismatch)";
143 case instrprof_error::counter_overflow:
144 OS <<
"counter overflow";
146 case instrprof_error::value_site_count_mismatch:
147 OS <<
"function value site count change detected (counter mismatch)";
149 case instrprof_error::compress_failed:
150 OS <<
"failed to compress data (zlib)";
152 case instrprof_error::uncompress_failed:
153 OS <<
"failed to uncompress data (zlib)";
155 case instrprof_error::empty_raw_profile:
156 OS <<
"empty raw profile file";
158 case instrprof_error::zlib_unavailable:
159 OS <<
"profile uses zlib compression but the profile reader was built "
160 "without zlib support";
162 case instrprof_error::raw_profile_version_mismatch:
163 OS <<
"raw profile version mismatch";
165 case instrprof_error::counter_value_too_large:
166 OS <<
"excessively large counter value suggests corrupted profile data";
172 OS <<
": " << ErrMsg;
182class InstrProfErrorCategoryType :
public std::error_category {
183 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
185 std::string message(
int IE)
const override {
193 static InstrProfErrorCategoryType ErrorCategory;
194 return ErrorCategory;
199const char *InstrProfSectNameCommon[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
205const char *InstrProfSectNameCoff[] = {
206#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
211const char *InstrProfSectNamePrefix[] = {
212#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
222 "enable-name-compression",
226 "enable-vtable-value-profiling",
cl::init(
false),
227 cl::desc(
"If true, the virtual table address will be instrumented to know "
228 "the types of a C++ pointer. The information is used in indirect "
229 "call promotion to do selective vtable-based comparison."));
233 bool AddSegmentInfo) {
234 std::string SectName;
237 SectName = InstrProfSectNamePrefix[IPSK];
240 SectName += InstrProfSectNameCoff[IPSK];
242 SectName += InstrProfSectNameCommon[IPSK];
244 if (OF ==
Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
245 SectName +=
",regular,live_support";
265 std::string NewName = std::string(
Name);
271 if (FileName.
empty())
272 NewName = NewName.insert(0,
"<unknown>:");
274 NewName = NewName.insert(0, FileName.
str() +
":");
285 for (
auto & CI : PathNameStr) {
294 return PathNameStr.
substr(LastPos);
345 MDNode *PGONameMetadata) {
353 return *IRPGOFuncName;
400 if (MangledName.empty())
401 return std::make_pair(
StringRef(), IRPGOName);
402 return std::make_pair(FileName, MangledName);
406 if (FileName.
empty())
426 const char InvalidChars[] =
"-:;<>/\"'";
427 size_t found = VarName.find_first_of(InvalidChars);
428 while (found != std::string::npos) {
429 VarName[found] =
'_';
430 found = VarName.find_first_of(InvalidChars, found + 1);
491 const uint8_t *EndP = NameStrings.
bytes_end();
498 bool isCompressed = (CompressedSize != 0);
506 UncompressedNameStrings,
512 NameStrings =
toStringRef(UncompressedNameStrings);
515 StringRef(
reinterpret_cast<const char *
>(
P), UncompressedSize);
516 P += UncompressedSize;
525 while (
P < EndP && *
P == 0)
541 this, std::placeholders::_1)))
552 CompressedVTableStrings,
566 const std::string UniqSuffix =
".__uniq.";
567 size_t pos = PGOName.
find(UniqSuffix);
569 pos += UniqSuffix.length();
575 pos = PGOName.
find(
'.', pos);
577 return PGOName.
substr(0, pos);
589 if (
Error E = mapName(PGOFuncName))
592 StringRef CanonicalFuncName = getCanonicalName(PGOFuncName);
593 if (CanonicalFuncName != PGOFuncName)
594 return mapName(CanonicalFuncName);
602 return VTableAddrMap.
lookup(Address, 0);
607 auto It =
partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t>
A) {
608 return A.first < Address;
614 if (It != AddrToMD5Map.end() && It->first == Address)
627 bool doCompression, std::string &Result) {
628 assert(!NameStrs.
empty() &&
"No name data to emit");
630 uint8_t Header[20], *
P = Header;
631 std::string UncompressedNameStrings =
636 "PGO name is invalid (contains separator token)");
638 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
641 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
644 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
645 unsigned HeaderLen =
P - &Header[0];
646 Result.append(HeaderStr, HeaderLen);
651 if (!doCompression) {
652 return WriteStringToResult(0, UncompressedNameStrings);
657 CompressedNameStrings,
660 return WriteStringToResult(CompressedNameStrings.
size(),
667 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
672 std::string &Result,
bool doCompression) {
673 std::vector<std::string> NameStrs;
674 for (
auto *NameVar : NameVars) {
682 std::string &Result,
bool doCompression) {
683 std::vector<std::string> VTableNameStrs;
684 for (
auto *VTable : VTables)
685 VTableNameStrs.push_back(
getPGOName(*VTable));
698 for (
uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
701 for (
size_t I = 0;
I < NumValueSites; ++
I) {
705 KindSum += VD[V].Count;
717 double Score = 0.0f, FuncLevelScore = 0.0f;
722 while (
I != IE && J != JE) {
723 if (
I->Value == J->Value) {
731 }
else if (
I->Value < J->Value) {
747 assert(ThisNumValueSites ==
Other.getNumValueSites(ValueKind));
748 if (!ThisNumValueSites)
751 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
752 getOrCreateValueSitesForKind(ValueKind);
754 Other.getValueSitesForKind(ValueKind);
756 ThisSiteRecords[
I].
overlap(OtherSiteRecords[
I], ValueKind, Overlap,
766 bool Mismatch = (
Counts.size() !=
Other.Counts.size());
770 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
773 if (ThisNumValueSites != OtherNumValueSites) {
785 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
791 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
794 MaxCount = std::max(
Other.Counts[
I], MaxCount);
799 if (MaxCount >= ValueCutoff) {
800 double FuncScore = 0.0;
801 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I)
807 FuncLevelOverlap.
Valid =
true;
818 for (
const InstrProfValueData &J : Input.
ValueData) {
819 while (
I != IE &&
I->Value < J.Value)
821 if (
I != IE &&
I->Value == J.Value) {
845void InstrProfRecord::mergeValueProfData(
849 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
850 if (ThisNumValueSites != OtherNumValueSites) {
854 if (!ThisNumValueSites)
856 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
857 getOrCreateValueSitesForKind(ValueKind);
859 Src.getValueSitesForKind(ValueKind);
861 ThisSiteRecords[
I].
merge(OtherSiteRecords[
I], Weight, Warn);
891 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
912 for (
size_t I = 0, E =
Other.BitmapBytes.size();
I < E; ++
I) {
916 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
917 mergeValueProfData(Kind,
Other, Weight, Warn);
920void InstrProfRecord::scaleValueProfData(
923 for (
auto &R : getValueSitesForKind(ValueKind))
929 assert(
D != 0 &&
"D cannot be 0");
930 for (
auto &Count : this->
Counts) {
940 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
941 scaleValueProfData(Kind,
N,
D, Warn);
950 if (ValueKind == IPVK_IndirectCallTarget)
953 if (ValueKind == IPVK_VTableTarget)
965 std::vector<InstrProfValueSiteRecord> &ValueSites =
966 getOrCreateValueSitesForKind(ValueKind);
968 ValueSites.emplace_back();
970 ValueSites.emplace_back(VData, VData +
N);
980 size_t LargestTraceSize = 0;
981 for (
auto &
Trace : Traces)
983 std::max(LargestTraceSize,
Trace.FunctionNameRefs.
size());
985 for (
auto &
Trace : Traces)
989 const int N =
Log2_64(LargestTraceSize) + 1;
994 for (
size_t TraceIdx = 0; TraceIdx < Traces.
size(); TraceIdx++) {
995 auto &
Trace = Traces[TraceIdx].FunctionNameRefs;
999 UtilityNodeT GroupId = TraceIdx *
N +
I;
1005 std::vector<BPFunctionNode> Nodes;
1006 for (
auto Id : FunctionIds) {
1007 auto &UNs = FuncGroups[Id];
1009 UNs.erase(std::unique(UNs.begin(), UNs.end()), UNs.end());
1010 Nodes.emplace_back(Id, UNs);
1015#define INSTR_PROF_COMMON_API_IMPL
1029 ->getNumValueSites(VKind);
1034 ->getNumValueData(VKind);
1040 ->getNumValueDataForSite(VK, S);
1045 reinterpret_cast<const InstrProfRecord *
>(R)->getValueForSite(Dst, K, S);
1050 (ValueProfData *)(
new (::operator
new(TotalSizeInBytes)) ValueProfData());
1051 memset(VD, 0, TotalSizeInBytes);
1068 Closure.Record = &
Record;
1069 return getValueProfDataSize(&Closure);
1073std::unique_ptr<ValueProfData>
1077 std::unique_ptr<ValueProfData> VPD(
1084 Record.reserveSites(Kind, NumValueSites);
1086 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
1087 for (
uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1088 uint8_t ValueDataCount = this->SiteCountArray[VSite];
1089 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
1090 ValueData += ValueDataCount;
1098 using namespace support;
1104 sys::swapByteOrder<uint32_t>(NumValueSites);
1105 sys::swapByteOrder<uint32_t>(Kind);
1107 uint32_t ND = getValueProfRecordNumValueData(
this);
1108 InstrProfValueData *VD = getValueProfRecordValueData(
this);
1112 sys::swapByteOrder<uint64_t>(VD[
I].
Value);
1113 sys::swapByteOrder<uint64_t>(VD[
I].Count);
1116 sys::swapByteOrder<uint32_t>(NumValueSites);
1117 sys::swapByteOrder<uint32_t>(Kind);
1123 if (NumValueKinds == 0)
1126 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1127 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1128 VR->deserializeTo(
Record, SymTab);
1129 VR = getValueProfRecordNext(VR);
1135 using namespace support;
1138 return endian::readNext<T, llvm::endianness::little>(
D);
1140 return endian::readNext<T, llvm::endianness::big>(
D);
1144 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
1148Error ValueProfData::checkIntegrity() {
1149 if (NumValueKinds > IPVK_Last + 1)
1150 return make_error<InstrProfError>(
1154 return make_error<InstrProfError>(
1157 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1158 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
1159 if (VR->Kind > IPVK_Last)
1161 "value kind is invalid");
1162 VR = getValueProfRecordNext(VR);
1163 if ((
char *)VR - (
char *)
this > (
ptrdiff_t)TotalSize)
1164 return make_error<InstrProfError>(
1166 "value profile address is greater than total size");
1172ValueProfData::getValueProfData(
const unsigned char *
D,
1173 const unsigned char *
const BufferEnd,
1175 using namespace support;
1177 if (
D +
sizeof(ValueProfData) > BufferEnd)
1180 const unsigned char *Header =
D;
1181 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
1182 if (
D + TotalSize > BufferEnd)
1186 memcpy(VPD.get(),
D, TotalSize);
1188 VPD->swapBytesToHost(Endianness);
1190 Error E = VPD->checkIntegrity();
1192 return std::move(E);
1194 return std::move(VPD);
1198 using namespace support;
1203 sys::swapByteOrder<uint32_t>(TotalSize);
1204 sys::swapByteOrder<uint32_t>(NumValueKinds);
1206 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1207 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1209 VR = getValueProfRecordNext(VR);
1214 using namespace support;
1219 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1220 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1221 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1225 sys::swapByteOrder<uint32_t>(TotalSize);
1226 sys::swapByteOrder<uint32_t>(NumValueKinds);
1238 std::unique_ptr<InstrProfValueData[]> VD =
1265 for (
auto &VD : VDs) {
1286 if (!
Tag || !
Tag->getString().equals(
"VP"))
1301 InstrProfValueData ValueData[],
1303 uint64_t &TotalC,
bool GetNoICPValue) {
1310 ActualNumValueData = 0;
1312 for (
unsigned I = 3;
I < NOps;
I += 2) {
1313 if (ActualNumValueData >= MaxNumDataWant)
1317 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1318 if (!
Value || !Count)
1323 ValueData[ActualNumValueData].Value =
Value->getZExtValue();
1324 ValueData[ActualNumValueData].Count = CntValue;
1325 ActualNumValueData++;
1330std::unique_ptr<InstrProfValueData[]>
1333 uint64_t &TotalC,
bool GetNoICPValue) {
1337 auto ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumValueData);
1339 ActualNumValueData, TotalC, GetNoICPValue))
1341 return ValueDataArray;
1349 InstrProfValueData ValueData[],
1351 bool GetNoICPValue) {
1356 ActualNumValueData, TotalC,
1366 if (PGOFuncName ==
F.getName())
1404 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1405 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1410 if (IRInstrVar->isDeclaration())
1414 if (!IRInstrVar->hasInitializer())
1417 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1420 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1425 if (
F.getName().empty())
1431 if (CheckAddressTaken &&
F.hasAddressTaken())
1439 if (!
F.hasComdat()) {
1448 if (InstrProfileOutput.
empty())
1454 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1456 Triple TT(M.getTargetTriple());
1457 if (TT.supportsCOMDAT()) {
1459 ProfileNameVar->
setComdat(M.getOrInsertComdat(
1460 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1465 const std::string &TestFilename,
1467 auto getProfileSum = [IsCS](
const std::string &Filename,
1473 if (
Error E = ReaderOrErr.takeError()) {
1476 auto Reader = std::move(ReaderOrErr.get());
1477 Reader->accumulateCounts(Sum, IsCS);
1495 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1505 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1515 const char *EntryName =
1516 (Level ==
ProgramLevel ?
"functions" :
"edge counters");
1518 OS <<
"Profile overlap infomation for base_profile: " << *
BaseFilename
1519 <<
" and test_profile: " << *
TestFilename <<
"\nProgram level:\n";
1521 OS <<
"Function level:\n"
1530 OS <<
" # of " << EntryName
1536 OS <<
" Mismatched count percentage (Edge): "
1539 OS <<
" Percentage of Edge profile only in test_profile: "
1546 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1549 char ProfileKindName[20] = {0};
1551 case IPVK_IndirectCallTarget:
1552 strncpy(ProfileKindName,
"IndirectCall", 19);
1554 case IPVK_MemOPSize:
1555 strncpy(ProfileKindName,
"MemOP", 19);
1557 case IPVK_VTableTarget:
1558 strncpy(ProfileKindName,
"VTable", 19);
1561 snprintf(ProfileKindName, 19,
"VP[%d]",
I);
1564 OS <<
" " << ProfileKindName
1568 OS <<
" Mismatched count percentage (" << ProfileKindName
1571 OS <<
" Percentage of " << ProfileKindName
1572 <<
" profile only in test_profile: "
1574 OS <<
" " << ProfileKindName
1577 <<
" " << ProfileKindName
1583namespace IndexedInstrProf {
1585template <
typename T1,
typename T2>
1587 constexpr T2 Object{};
1588 return size_t(&(Object.*Member)) - size_t(&Object);
1596 using namespace support;
1597 return endian::byte_swap<uint64_t, llvm::endianness::little>(
Version);
1601 using namespace support;
1602 static_assert(std::is_standard_layout_v<Header>,
1603 "The header should be standard layout type since we use offset "
1604 "of fields to read.");
1610 endian::byte_swap<uint64_t, llvm::endianness::little>(
H.Magic);
1616 if (GET_VERSION(
H.formatVersion()) >
1620 switch (GET_VERSION(
H.formatVersion())) {
1625 "Please update the reading code below if a new field has been added, "
1626 "if not add a case statement to fall through to the latest version.");
1633 H.TemporalProfTracesOffset =
1656 "Please update the size computation below if a new field has "
1657 "been added to the header, if not add a case statement to "
1658 "fall through to the latest version.");
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
void setComdat(Comdat *C)
static bool isLocalLinkage(LinkageTypes Linkage)
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Module * getParent()
Get the module that this global value is contained inside of...
bool isDiscardableIfUnused() const
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ ExternalWeakLinkage
ExternalWeak linkage description.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
std::string message() const override
Return the error message as a string.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
uint64_t getVTableHashFromAddress(uint64_t Address)
Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
Error addVTableName(StringRef VTableName)
Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
The method name is kept since there are many callers.
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)
Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
ValT lookup(KeyT x, ValT NotFound=ValT()) const
lookup - Return the mapped value at x or NotFound.
This is an important class for using LLVM in a threaded context.
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
MDString * createString(StringRef Str)
Return the given string as metadata.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
const std::string & getSourceFileName() const
Get the module's original source file name.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
A vector that has set insertion semantics.
bool insert(const value_type &X)
Insert a new element into the SetVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string str() const
str - Get the contents as an std::string.
const unsigned char * bytes_end() const
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
static constexpr size_t npos
const unsigned char * bytes_begin() const
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This class represents a function that is read from a sample profile.
@ C
The default llvm calling convention, compatible with C.
static uint64_t read(const unsigned char *Buffer, size_t Offset)
size_t constexpr offsetOf(T1 T2::*Member)
initializer< Ty > init(const Ty &Val)
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
StringRef getPGOFuncNameMetadataName()
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
MDNode * mayHaveValueProfileOfKind(const Instruction &Inst, InstrProfValueKind ValueKind)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
constexpr char kGlobalIdentifierDelimiter
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void sort(IteratorTy Start, IteratorTy End)
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig)
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata)
@ value_site_count_mismatch
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
const std::error_category & instrprof_category()
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
static StringRef getStrippedSourceFileName(const GlobalObject &GO)
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
static bool getValueProfDataFromInstImpl(const MDNode *const MD, const uint32_t MaxNumDataWant, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue)
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
static ValueProfRecordClosure InstrProfRecordClosure
static Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)
NameStrings is a string composed of one of more possibly encoded sub-strings.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
static std::optional< std::string > lookupPGONameFromMetadata(MDNode *MD)
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
void consumeError(Error Err)
Consume a Error without doing anything.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName)
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
double ValueCounts[IPVK_Last - IPVK_First+1]
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
void setPseudoCount(CountPseudoKind Kind)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
std::vector< uint8_t > BitmapBytes
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
CountSumOrPercent Mismatch
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...