21#include "llvm/Config/config.h"
55#include <system_error>
64 cl::desc(
"Use full module build paths in the profile counter names for "
65 "static functions."));
77 cl::desc(
"Strip specified level of directory name from source path in "
78 "the profile counter name for static functions."));
81 const std::string &ErrMsg =
"") {
86 case instrprof_error::success:
89 case instrprof_error::eof:
92 case instrprof_error::unrecognized_format:
93 OS <<
"unrecognized instrumentation profile encoding format";
95 case instrprof_error::bad_magic:
96 OS <<
"invalid instrumentation profile data (bad magic)";
98 case instrprof_error::bad_header:
99 OS <<
"invalid instrumentation profile data (file header is corrupt)";
101 case instrprof_error::unsupported_version:
102 OS <<
"unsupported instrumentation profile format version";
104 case instrprof_error::unsupported_hash_type:
105 OS <<
"unsupported instrumentation profile hash type";
107 case instrprof_error::too_large:
108 OS <<
"too much profile data";
110 case instrprof_error::truncated:
111 OS <<
"truncated profile data";
113 case instrprof_error::malformed:
114 OS <<
"malformed instrumentation profile data";
116 case instrprof_error::missing_debug_info_for_correlation:
117 OS <<
"debug info for correlation is required";
119 case instrprof_error::unexpected_debug_info_for_correlation:
120 OS <<
"debug info for correlation is not necessary";
122 case instrprof_error::unable_to_correlate_profile:
123 OS <<
"unable to correlate profile";
125 case instrprof_error::invalid_prof:
126 OS <<
"invalid profile created. Please file a bug "
127 "at: " BUG_REPORT_URL
128 " and include the profraw files that caused this error.";
130 case instrprof_error::unknown_function:
131 OS <<
"no profile data available for function";
133 case instrprof_error::hash_mismatch:
134 OS <<
"function control flow change detected (hash mismatch)";
136 case instrprof_error::count_mismatch:
137 OS <<
"function basic block count change detected (counter mismatch)";
139 case instrprof_error::counter_overflow:
140 OS <<
"counter overflow";
142 case instrprof_error::value_site_count_mismatch:
143 OS <<
"function value site count change detected (counter mismatch)";
145 case instrprof_error::compress_failed:
146 OS <<
"failed to compress data (zlib)";
148 case instrprof_error::uncompress_failed:
149 OS <<
"failed to uncompress data (zlib)";
151 case instrprof_error::empty_raw_profile:
152 OS <<
"empty raw profile file";
154 case instrprof_error::zlib_unavailable:
155 OS <<
"profile uses zlib compression but the profile reader was built "
156 "without zlib support";
158 case instrprof_error::raw_profile_version_mismatch:
159 OS <<
"raw profile version mismatch";
165 OS <<
": " << ErrMsg;
175class InstrProfErrorCategoryType :
public std::error_category {
176 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
178 std::string message(
int IE)
const override {
186 static InstrProfErrorCategoryType ErrorCategory;
187 return ErrorCategory;
192const char *InstrProfSectNameCommon[] = {
193#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
198const char *InstrProfSectNameCoff[] = {
199#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
204const char *InstrProfSectNamePrefix[] = {
205#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
215 "enable-name-compression",
220 bool AddSegmentInfo) {
221 std::string SectName;
224 SectName = InstrProfSectNamePrefix[IPSK];
227 SectName += InstrProfSectNameCoff[IPSK];
229 SectName += InstrProfSectNameCommon[IPSK];
231 if (OF ==
Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
232 SectName +=
",regular,live_support";
256 for (
auto & CI : PathNameStr) {
265 return PathNameStr.
substr(LastPos);
269 StringRef FileName(
F.getParent()->getSourceFileName());
297 Name.append(FileName.
empty() ?
"<unknown>" : FileName);
301 return Name.str().str();
306 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
321 return *IRPGOFuncName;
359std::pair<StringRef, StringRef>
361 auto [FileName, FuncName] = IRPGOFuncName.
split(
';');
362 if (FuncName.empty())
363 return std::make_pair(
StringRef(), IRPGOFuncName);
364 return std::make_pair(FileName, FuncName);
368 if (FileName.
empty())
387 const char InvalidChars[] =
"-:;<>/\"'";
388 size_t found = VarName.find_first_of(InvalidChars);
389 while (found != std::string::npos) {
390 VarName[found] =
'_';
391 found = VarName.find_first_of(InvalidChars, found + 1);
458 const std::string UniqSuffix =
".__uniq.";
459 auto pos = PGOFuncName.
find(UniqSuffix);
462 if (pos != std::string::npos)
463 pos += UniqSuffix.length();
466 pos = PGOFuncName.
find(
'.', pos);
467 if (pos != std::string::npos && pos != 0) {
478 auto It =
partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t>
A) {
479 return A.first < Address;
485 if (It != AddrToMD5Map.end() && It->first == Address)
498 bool doCompression, std::string &Result) {
499 assert(!NameStrs.
empty() &&
"No name data to emit");
501 uint8_t Header[20], *
P = Header;
502 std::string UncompressedNameStrings =
507 "PGO name is invalid (contains separator token)");
509 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
512 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
515 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
516 unsigned HeaderLen =
P - &Header[0];
517 Result.append(HeaderStr, HeaderLen);
522 if (!doCompression) {
523 return WriteStringToResult(0, UncompressedNameStrings);
528 CompressedNameStrings,
531 return WriteStringToResult(CompressedNameStrings.
size(),
538 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
543 std::string &Result,
bool doCompression) {
544 std::vector<std::string> NameStrs;
545 for (
auto *NameVar : NameVars) {
554 const uint8_t *EndP = NameStrings.
bytes_end();
561 bool isCompressed = (CompressedSize != 0);
569 UncompressedNameStrings,
575 NameStrings =
toStringRef(UncompressedNameStrings);
578 StringRef(
reinterpret_cast<const char *
>(
P), UncompressedSize);
579 P += UncompressedSize;
588 while (
P < EndP && *
P == 0)
601 for (
uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
604 for (
size_t I = 0;
I < NumValueSites; ++
I) {
608 KindSum += VD[V].Count;
620 double Score = 0.0f, FuncLevelScore = 0.0f;
625 while (
I != IE && J != JE) {
626 if (
I->Value == J->Value) {
634 }
else if (
I->Value < J->Value) {
650 assert(ThisNumValueSites ==
Other.getNumValueSites(ValueKind));
651 if (!ThisNumValueSites)
654 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
655 getOrCreateValueSitesForKind(ValueKind);
657 Other.getValueSitesForKind(ValueKind);
659 ThisSiteRecords[
I].
overlap(OtherSiteRecords[
I], ValueKind, Overlap,
669 bool Mismatch = (
Counts.size() !=
Other.Counts.size());
673 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
676 if (ThisNumValueSites != OtherNumValueSites) {
688 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
694 for (
size_t I = 0,
E =
Other.Counts.size();
I <
E; ++
I) {
697 MaxCount = std::max(
Other.Counts[
I], MaxCount);
702 if (MaxCount >= ValueCutoff) {
703 double FuncScore = 0.0;
704 for (
size_t I = 0,
E =
Other.Counts.size();
I <
E; ++
I)
710 FuncLevelOverlap.
Valid =
true;
721 for (
const InstrProfValueData &J : Input.
ValueData) {
722 while (
I != IE &&
I->Value < J.Value)
724 if (
I != IE &&
I->Value == J.Value) {
748void InstrProfRecord::mergeValueProfData(
752 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
753 if (ThisNumValueSites != OtherNumValueSites) {
757 if (!ThisNumValueSites)
759 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
760 getOrCreateValueSitesForKind(ValueKind);
762 Src.getValueSitesForKind(ValueKind);
764 ThisSiteRecords[
I].
merge(OtherSiteRecords[
I], Weight, Warn);
794 for (
size_t I = 0,
E =
Other.Counts.size();
I <
E; ++
I) {
807 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
808 mergeValueProfData(Kind,
Other, Weight, Warn);
811void InstrProfRecord::scaleValueProfData(
814 for (
auto &R : getValueSitesForKind(ValueKind))
820 assert(
D != 0 &&
"D cannot be 0");
821 for (
auto &Count : this->
Counts) {
831 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
832 scaleValueProfData(Kind,
N,
D, Warn);
841 if (ValueKind == IPVK_IndirectCallTarget)
853 std::vector<InstrProfValueSiteRecord> &ValueSites =
854 getOrCreateValueSitesForKind(ValueKind);
856 ValueSites.emplace_back();
858 ValueSites.emplace_back(VData, VData +
N);
868 size_t LargestTraceSize = 0;
869 for (
auto &
Trace : Traces)
871 std::max(LargestTraceSize,
Trace.FunctionNameRefs.
size());
873 for (
auto &
Trace : Traces)
877 int N = std::ceil(std::log2(LargestTraceSize));
882 for (
size_t TraceIdx = 0; TraceIdx < Traces.
size(); TraceIdx++) {
883 auto &
Trace = Traces[TraceIdx].FunctionNameRefs;
885 for (
int I = std::floor(std::log2(
Timestamp + 1));
I <
N;
I++) {
887 UtilityNodeT GroupId = TraceIdx *
N +
I;
888 FuncGroups[FunctionId].push_back(GroupId);
893 std::vector<BPFunctionNode> Nodes;
894 for (
auto &Id : FunctionIds) {
895 auto &UNs = FuncGroups[Id];
897 UNs.erase(std::unique(UNs.begin(), UNs.end()), UNs.end());
898 Nodes.emplace_back(Id, UNs);
903#define INSTR_PROF_COMMON_API_IMPL
917 ->getNumValueSites(VKind);
922 ->getNumValueData(VKind);
928 ->getNumValueDataForSite(VK, S);
933 reinterpret_cast<const InstrProfRecord *
>(R)->getValueForSite(Dst, K, S);
938 (ValueProfData *)(
new (::operator
new(TotalSizeInBytes)) ValueProfData());
939 memset(VD, 0, TotalSizeInBytes);
957 return getValueProfDataSize(&Closure);
961std::unique_ptr<ValueProfData>
965 std::unique_ptr<ValueProfData> VPD(
972 Record.reserveSites(Kind, NumValueSites);
974 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
975 for (
uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
976 uint8_t ValueDataCount = this->SiteCountArray[VSite];
977 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
978 ValueData += ValueDataCount;
987 using namespace support;
993 sys::swapByteOrder<uint32_t>(NumValueSites);
994 sys::swapByteOrder<uint32_t>(Kind);
996 uint32_t ND = getValueProfRecordNumValueData(
this);
997 InstrProfValueData *VD = getValueProfRecordValueData(
this);
1001 sys::swapByteOrder<uint64_t>(VD[
I].
Value);
1002 sys::swapByteOrder<uint64_t>(VD[
I].Count);
1005 sys::swapByteOrder<uint32_t>(NumValueSites);
1006 sys::swapByteOrder<uint32_t>(Kind);
1012 if (NumValueKinds == 0)
1015 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1016 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1017 VR->deserializeTo(
Record, SymTab);
1018 VR = getValueProfRecordNext(VR);
1024 using namespace support;
1027 return endian::readNext<T, little, unaligned>(
D);
1029 return endian::readNext<T, big, unaligned>(
D);
1033 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
1037Error ValueProfData::checkIntegrity() {
1038 if (NumValueKinds > IPVK_Last + 1)
1039 return make_error<InstrProfError>(
1043 return make_error<InstrProfError>(
1046 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1047 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
1048 if (VR->Kind > IPVK_Last)
1050 "value kind is invalid");
1051 VR = getValueProfRecordNext(VR);
1052 if ((
char *)VR - (
char *)
this > (
ptrdiff_t)TotalSize)
1053 return make_error<InstrProfError>(
1055 "value profile address is greater than total size");
1061ValueProfData::getValueProfData(
const unsigned char *
D,
1062 const unsigned char *
const BufferEnd,
1064 using namespace support;
1066 if (
D +
sizeof(ValueProfData) > BufferEnd)
1069 const unsigned char *Header =
D;
1070 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
1071 if (
D + TotalSize > BufferEnd)
1075 memcpy(VPD.get(),
D, TotalSize);
1077 VPD->swapBytesToHost(Endianness);
1079 Error E = VPD->checkIntegrity();
1081 return std::move(
E);
1083 return std::move(VPD);
1087 using namespace support;
1092 sys::swapByteOrder<uint32_t>(TotalSize);
1093 sys::swapByteOrder<uint32_t>(NumValueKinds);
1095 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1096 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1098 VR = getValueProfRecordNext(VR);
1103 using namespace support;
1108 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1109 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1110 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1114 sys::swapByteOrder<uint32_t>(TotalSize);
1115 sys::swapByteOrder<uint32_t>(NumValueKinds);
1127 std::unique_ptr<InstrProfValueData[]> VD =
1152 for (
auto &VD : VDs) {
1166 InstrProfValueData ValueData[],
1168 bool GetNoICPValue) {
1183 if (!
Tag->getString().equals(
"VP"))
1199 ActualNumValueData = 0;
1201 for (
unsigned I = 3;
I < NOps;
I += 2) {
1202 if (ActualNumValueData >= MaxNumValueData)
1206 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1207 if (!
Value || !Count)
1212 ValueData[ActualNumValueData].Value =
Value->getZExtValue();
1213 ValueData[ActualNumValueData].Count = CntValue;
1214 ActualNumValueData++;
1225 if (PGOFuncName ==
F.getName())
1263 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1264 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1269 if (IRInstrVar->isDeclaration())
1273 if (!IRInstrVar->hasInitializer())
1276 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1279 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1284 if (
F.getName().empty())
1290 if (CheckAddressTaken &&
F.hasAddressTaken())
1298 if (!
F.hasComdat()) {
1307 if (InstrProfileOutput.
empty())
1313 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1315 Triple TT(M.getTargetTriple());
1316 if (TT.supportsCOMDAT()) {
1318 ProfileNameVar->
setComdat(M.getOrInsertComdat(
1319 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1324 const std::string &TestFilename,
1326 auto getProfileSum = [IsCS](
const std::string &Filename,
1332 if (
Error E = ReaderOrErr.takeError()) {
1335 auto Reader = std::move(ReaderOrErr.get());
1336 Reader->accumulateCounts(Sum, IsCS);
1354 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1364 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1374 const char *EntryName =
1375 (Level ==
ProgramLevel ?
"functions" :
"edge counters");
1377 OS <<
"Profile overlap infomation for base_profile: " << *
BaseFilename
1378 <<
" and test_profile: " << *
TestFilename <<
"\nProgram level:\n";
1380 OS <<
"Function level:\n"
1389 OS <<
" # of " << EntryName
1395 OS <<
" Mismatched count percentage (Edge): "
1398 OS <<
" Percentage of Edge profile only in test_profile: "
1405 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1408 char ProfileKindName[20];
1410 case IPVK_IndirectCallTarget:
1411 strncpy(ProfileKindName,
"IndirectCall", 19);
1413 case IPVK_MemOPSize:
1414 strncpy(ProfileKindName,
"MemOP", 19);
1417 snprintf(ProfileKindName, 19,
"VP[%d]",
I);
1420 OS <<
" " << ProfileKindName
1424 OS <<
" Mismatched count percentage (" << ProfileKindName
1427 OS <<
" Percentage of " << ProfileKindName
1428 <<
" profile only in test_profile: "
1430 OS <<
" " << ProfileKindName
1433 <<
" " << ProfileKindName
1439namespace IndexedInstrProf {
1441template <
typename T1,
typename T2>
1443 constexpr T2 Object{};
1444 return size_t(&(Object.*Member)) - size_t(&Object);
1452 using namespace support;
1453 return endian::byte_swap<uint64_t, little>(
Version);
1457 using namespace support;
1458 static_assert(std::is_standard_layout_v<Header>,
1459 "The header should be standard layout type since we use offset "
1460 "of fields to read.");
1471 if (GET_VERSION(
H.formatVersion()) >
1475 switch (GET_VERSION(
H.formatVersion())) {
1480 "Please update the reading code below if a new field has been added, "
1481 "if not add a case statement to fall through to the latest version.");
1483 H.TemporalProfTracesOffset =
1506 "Please update the size computation below if a new field has "
1507 "been added to the header, if not add a case statement to "
1508 "fall through to the latest version.");
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallString class.
This file defines the SmallVector class.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
This is the shared class of boolean and integer constants.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
void setComdat(Comdat *C)
static bool isLocalLinkage(LinkageTypes Linkage)
void setLinkage(LinkageTypes LT)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
bool isDiscardableIfUnused() const
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ ExternalWeakLinkage
ExternalWeak linkage description.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
std::string message() const override
Return the error message as a string.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
This is an important class for using LLVM in a threaded context.
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
MDString * createString(StringRef Str)
Return the given string as metadata.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable's name.
A Module instance is used to store all the information related to an LLVM module.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
A vector that has set insertion semantics.
bool insert(const value_type &X)
Insert a new element into the SetVector.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string str() const
str - Get the contents as an std::string.
const unsigned char * bytes_end() const
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
bool startswith(StringRef Prefix) const
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
const unsigned char * bytes_begin() const
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
@ C
The default llvm calling convention, compatible with C.
static uint64_t read(const unsigned char *Buffer, size_t Offset)
size_t constexpr offsetOf(T1 T2::*Member)
initializer< Ty > init(const Ty &Val)
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
StringRef getPGOFuncNameMetadataName()
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig)
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
std::pair< StringRef, StringRef > getParsedIRPGOFuncName(StringRef IRPGOFuncName)
void sort(IteratorTy Start, IteratorTy End)
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
support::endianness getHostEndianness()
static std::optional< std::string > lookupPGOFuncName(const Function &F)
@ value_site_count_mismatch
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
const std::error_category & instrprof_category()
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
static ValueProfRecordClosure InstrProfRecordClosure
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
static StringRef getStrippedSourceFileName(const Function &F)
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
void consumeError(Error Err)
Consume a Error without doing anything.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
double ValueCounts[IPVK_Last - IPVK_First+1]
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
void setPseudoCount(CountPseudoKind Kind)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
CountSumOrPercent Mismatch
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...