20#include "llvm/Config/config.h"
54#include <system_error>
61#define DEBUG_TYPE "instrprof"
65 cl::desc(
"Use full module build paths in the profile counter names for "
66 "static functions."));
78 cl::desc(
"Strip specified level of directory name from source path in "
79 "the profile counter name for static functions."));
82 const std::string &ErrMsg =
"") {
87 case instrprof_error::success:
90 case instrprof_error::eof:
93 case instrprof_error::unrecognized_format:
94 OS <<
"unrecognized instrumentation profile encoding format";
96 case instrprof_error::bad_magic:
97 OS <<
"invalid instrumentation profile data (bad magic)";
99 case instrprof_error::bad_header:
100 OS <<
"invalid instrumentation profile data (file header is corrupt)";
102 case instrprof_error::unsupported_version:
103 OS <<
"unsupported instrumentation profile format version";
105 case instrprof_error::unsupported_hash_type:
106 OS <<
"unsupported instrumentation profile hash type";
108 case instrprof_error::too_large:
109 OS <<
"too much profile data";
111 case instrprof_error::truncated:
112 OS <<
"truncated profile data";
114 case instrprof_error::malformed:
115 OS <<
"malformed instrumentation profile data";
117 case instrprof_error::missing_correlation_info:
118 OS <<
"debug info/binary for correlation is required";
120 case instrprof_error::unexpected_correlation_info:
121 OS <<
"debug info/binary for correlation is not necessary";
123 case instrprof_error::unable_to_correlate_profile:
124 OS <<
"unable to correlate profile";
126 case instrprof_error::invalid_prof:
127 OS <<
"invalid profile created. Please file a bug "
128 "at: " BUG_REPORT_URL
129 " and include the profraw files that caused this error.";
131 case instrprof_error::unknown_function:
132 OS <<
"no profile data available for function";
134 case instrprof_error::hash_mismatch:
135 OS <<
"function control flow change detected (hash mismatch)";
137 case instrprof_error::count_mismatch:
138 OS <<
"function basic block count change detected (counter mismatch)";
140 case instrprof_error::bitmap_mismatch:
141 OS <<
"function bitmap size change detected (bitmap size mismatch)";
143 case instrprof_error::counter_overflow:
144 OS <<
"counter overflow";
146 case instrprof_error::value_site_count_mismatch:
147 OS <<
"function value site count change detected (counter mismatch)";
149 case instrprof_error::compress_failed:
150 OS <<
"failed to compress data (zlib)";
152 case instrprof_error::uncompress_failed:
153 OS <<
"failed to uncompress data (zlib)";
155 case instrprof_error::empty_raw_profile:
156 OS <<
"empty raw profile file";
158 case instrprof_error::zlib_unavailable:
159 OS <<
"profile uses zlib compression but the profile reader was built "
160 "without zlib support";
162 case instrprof_error::raw_profile_version_mismatch:
163 OS <<
"raw profile version mismatch";
165 case instrprof_error::counter_value_too_large:
166 OS <<
"excessively large counter value suggests corrupted profile data";
172 OS <<
": " << ErrMsg;
182class InstrProfErrorCategoryType :
public std::error_category {
183 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
185 std::string message(
int IE)
const override {
193 static InstrProfErrorCategoryType ErrorCategory;
194 return ErrorCategory;
199const char *InstrProfSectNameCommon[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
205const char *InstrProfSectNameCoff[] = {
206#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
211const char *InstrProfSectNamePrefix[] = {
212#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
222 "enable-name-compression",
226 "enable-vtable-value-profiling",
cl::init(
false),
227 cl::desc(
"If true, the virtual table address will be instrumented to know "
228 "the types of a C++ pointer. The information is used in indirect "
229 "call promotion to do selective vtable-based comparison."));
232 "enable-vtable-profile-use",
cl::init(
false),
233 cl::desc(
"If ThinLTO and WPD is enabled and this option is true, vtable "
234 "profiles will be used by ICP pass for more efficient indirect "
235 "call sequence. If false, type profiles won't be used."));
239 bool AddSegmentInfo) {
240 std::string SectName;
243 SectName = InstrProfSectNamePrefix[IPSK];
246 SectName += InstrProfSectNameCoff[IPSK];
248 SectName += InstrProfSectNameCommon[IPSK];
250 if (OF ==
Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
251 SectName +=
",regular,live_support";
271 std::string NewName = std::string(
Name);
277 if (FileName.
empty())
278 NewName = NewName.insert(0,
"<unknown>:");
280 NewName = NewName.insert(0, FileName.
str() +
":");
291 for (
const auto &CI : PathNameStr) {
300 return PathNameStr.
substr(LastPos);
351 MDNode *PGONameMetadata) {
359 return *IRPGOFuncName;
406 if (MangledName.empty())
407 return std::make_pair(
StringRef(), IRPGOName);
408 return std::make_pair(FileName, MangledName);
412 if (FileName.
empty())
432 const char InvalidChars[] =
"-:;<>/\"'";
433 size_t found = VarName.find_first_of(InvalidChars);
434 while (found != std::string::npos) {
435 VarName[found] =
'_';
436 found = VarName.find_first_of(InvalidChars, found + 1);
487 if (!
G.hasName() || !
G.hasMetadata(LLVMContext::MD_type))
504 bool Inserted =
true;
505 std::tie(std::ignore, Inserted) =
511 if (
Error E = mapName(VTablePGOName))
514 StringRef CanonicalName = getCanonicalName(VTablePGOName);
515 if (CanonicalName != VTablePGOName)
516 return mapName(CanonicalName);
528 const uint8_t *EndP = NameStrings.
bytes_end();
535 bool isCompressed = (CompressedSize != 0);
543 UncompressedNameStrings,
549 NameStrings =
toStringRef(UncompressedNameStrings);
552 StringRef(
reinterpret_cast<const char *
>(
P), UncompressedSize);
553 P += UncompressedSize;
562 while (
P < EndP && *
P == 0)
578 this, std::placeholders::_1)))
589 CompressedVTableStrings,
603 const std::string UniqSuffix =
".__uniq.";
604 size_t pos = PGOName.
find(UniqSuffix);
606 pos += UniqSuffix.length();
612 pos = PGOName.
find(
'.', pos);
614 return PGOName.
substr(0, pos);
626 if (
Error E = mapName(PGOFuncName))
629 StringRef CanonicalFuncName = getCanonicalName(PGOFuncName);
630 if (CanonicalFuncName != PGOFuncName)
631 return mapName(CanonicalFuncName);
639 return VTableAddrMap.
lookup(Address, 0);
644 auto It =
partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t>
A) {
645 return A.first < Address;
651 if (It != AddrToMD5Map.end() && It->first == Address)
664 bool doCompression, std::string &Result) {
665 assert(!NameStrs.
empty() &&
"No name data to emit");
667 uint8_t Header[20], *
P = Header;
668 std::string UncompressedNameStrings =
673 "PGO name is invalid (contains separator token)");
675 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
678 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
681 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
682 unsigned HeaderLen =
P - &Header[0];
683 Result.append(HeaderStr, HeaderLen);
688 if (!doCompression) {
689 return WriteStringToResult(0, UncompressedNameStrings);
694 CompressedNameStrings,
697 return WriteStringToResult(CompressedNameStrings.
size(),
704 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
709 std::string &Result,
bool doCompression) {
710 std::vector<std::string> NameStrs;
711 for (
auto *NameVar : NameVars) {
719 std::string &Result,
bool doCompression) {
720 std::vector<std::string> VTableNameStrs;
721 for (
auto *VTable : VTables)
722 VTableNameStrs.push_back(
getPGOName(*VTable));
735 for (
uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
738 for (
size_t I = 0;
I < NumValueSites; ++
I) {
752 double Score = 0.0f, FuncLevelScore = 0.0f;
757 while (
I != IE && J != JE) {
758 if (
I->Value == J->Value) {
766 }
else if (
I->Value < J->Value) {
782 assert(ThisNumValueSites ==
Other.getNumValueSites(ValueKind));
783 if (!ThisNumValueSites)
786 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
787 getOrCreateValueSitesForKind(ValueKind);
789 Other.getValueSitesForKind(ValueKind);
791 ThisSiteRecords[
I].
overlap(OtherSiteRecords[
I], ValueKind, Overlap,
801 bool Mismatch = (
Counts.size() !=
Other.Counts.size());
805 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
808 if (ThisNumValueSites != OtherNumValueSites) {
820 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
826 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
829 MaxCount = std::max(
Other.Counts[
I], MaxCount);
834 if (MaxCount >= ValueCutoff) {
835 double FuncScore = 0.0;
836 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I)
842 FuncLevelOverlap.
Valid =
true;
853 std::vector<InstrProfValueData> Merged;
855 for (
const InstrProfValueData &J : Input.
ValueData) {
856 while (
I != IE &&
I->Value < J.Value) {
857 Merged.push_back(*
I);
860 if (
I != IE &&
I->Value == J.Value) {
865 Merged.push_back(*
I);
871 Merged.insert(Merged.end(),
I, IE);
887void InstrProfRecord::mergeValueProfData(
891 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
892 if (ThisNumValueSites != OtherNumValueSites) {
896 if (!ThisNumValueSites)
898 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
899 getOrCreateValueSitesForKind(ValueKind);
901 Src.getValueSitesForKind(ValueKind);
903 ThisSiteRecords[
I].
merge(OtherSiteRecords[
I], Weight, Warn);
933 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
954 for (
size_t I = 0, E =
Other.BitmapBytes.size();
I < E; ++
I) {
958 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
959 mergeValueProfData(Kind,
Other, Weight, Warn);
962void InstrProfRecord::scaleValueProfData(
965 for (
auto &R : getValueSitesForKind(ValueKind))
971 assert(
D != 0 &&
"D cannot be 0");
972 for (
auto &Count : this->
Counts) {
982 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
983 scaleValueProfData(Kind,
N,
D, Warn);
992 if (ValueKind == IPVK_IndirectCallTarget)
995 if (ValueKind == IPVK_VTableTarget)
1007 std::vector<InstrProfValueSiteRecord> &ValueSites =
1008 getOrCreateValueSitesForKind(ValueKind);
1009 assert(ValueSites.size() == Site);
1011 ValueSites.emplace_back();
1013 ValueSites.emplace_back(VData, VData +
N);
1018 bool RemoveOutlierUNs) {
1021 UtilityNodeT MaxUN = 0;
1027 for (
auto &
Trace : Traces) {
1028 size_t CutoffTimestamp = 1;
1034 It->getSecond() = std::min<size_t>(It->getSecond(),
Timestamp);
1041 for (
auto &[Id, FirstUN] : IdToFirstUN)
1042 for (
auto UN = FirstUN; UN <= MaxUN; ++UN)
1043 IdToUNs[Id].push_back(UN);
1045 IdToFirstUN.
clear();
1048 if (RemoveOutlierUNs) {
1050 for (
auto &[Id, UNs] : IdToUNs)
1051 for (
auto &UN : UNs)
1055 for (
auto &[Id, UNs] : IdToUNs)
1057 return UNFrequency[UN] <= 1 || 2 * UNFrequency[UN] > IdToUNs.
size();
1061 for (
auto &[Id, UNs] : IdToUNs)
1062 Nodes.emplace_back(Id, UNs);
1067 return std::make_pair(IdToFirstTimestamp[L.Id], L.Id) <
1068 std::make_pair(IdToFirstTimestamp[R.Id], R.Id);
1072#define INSTR_PROF_COMMON_API_IMPL
1086 ->getNumValueSites(VKind);
1091 ->getNumValueData(VKind);
1103 llvm::copy(IPR->getValueArrayForSite(K, S), Dst);
1108 (ValueProfData *)(
new (::operator
new(TotalSizeInBytes)) ValueProfData());
1109 memset(VD, 0, TotalSizeInBytes);
1126 Closure.Record = &
Record;
1127 return getValueProfDataSize(&Closure);
1131std::unique_ptr<ValueProfData>
1135 std::unique_ptr<ValueProfData> VPD(
1142 Record.reserveSites(Kind, NumValueSites);
1144 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
1145 for (
uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1146 uint8_t ValueDataCount = this->SiteCountArray[VSite];
1147 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
1148 ValueData += ValueDataCount;
1156 using namespace support;
1162 sys::swapByteOrder<uint32_t>(NumValueSites);
1163 sys::swapByteOrder<uint32_t>(Kind);
1165 uint32_t ND = getValueProfRecordNumValueData(
this);
1166 InstrProfValueData *VD = getValueProfRecordValueData(
this);
1170 sys::swapByteOrder<uint64_t>(VD[
I].
Value);
1171 sys::swapByteOrder<uint64_t>(VD[
I].Count);
1174 sys::swapByteOrder<uint32_t>(NumValueSites);
1175 sys::swapByteOrder<uint32_t>(Kind);
1181 if (NumValueKinds == 0)
1184 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1185 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1186 VR->deserializeTo(
Record, SymTab);
1187 VR = getValueProfRecordNext(VR);
1192 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
1196Error ValueProfData::checkIntegrity() {
1197 if (NumValueKinds > IPVK_Last + 1)
1198 return make_error<InstrProfError>(
1202 return make_error<InstrProfError>(
1205 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1206 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
1207 if (VR->Kind > IPVK_Last)
1209 "value kind is invalid");
1210 VR = getValueProfRecordNext(VR);
1211 if ((
char *)VR - (
char *)
this > (
ptrdiff_t)TotalSize)
1212 return make_error<InstrProfError>(
1214 "value profile address is greater than total size");
1220ValueProfData::getValueProfData(
const unsigned char *
D,
1221 const unsigned char *
const BufferEnd,
1223 using namespace support;
1225 if (
D +
sizeof(ValueProfData) > BufferEnd)
1228 const unsigned char *Header =
D;
1229 uint32_t TotalSize = endian::readNext<uint32_t>(Header, Endianness);
1231 if (
D + TotalSize > BufferEnd)
1235 memcpy(VPD.get(),
D, TotalSize);
1237 VPD->swapBytesToHost(Endianness);
1239 Error E = VPD->checkIntegrity();
1241 return std::move(E);
1243 return std::move(VPD);
1247 using namespace support;
1252 sys::swapByteOrder<uint32_t>(TotalSize);
1253 sys::swapByteOrder<uint32_t>(NumValueKinds);
1255 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1256 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1258 VR = getValueProfRecordNext(VR);
1263 using namespace support;
1268 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1269 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1270 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1274 sys::swapByteOrder<uint32_t>(TotalSize);
1275 sys::swapByteOrder<uint32_t>(NumValueKinds);
1286 for (
const InstrProfValueData &V : VDs)
1311 for (
const auto &VD : VDs) {
1332 if (!
Tag ||
Tag->getString() !=
"VP")
1347 InstrProfValueData ValueData[],
1349 uint64_t &TotalC,
bool GetNoICPValue) {
1356 ActualNumValueData = 0;
1358 for (
unsigned I = 3;
I < NOps;
I += 2) {
1359 if (ActualNumValueData >= MaxNumDataWant)
1363 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1364 if (!
Value || !Count)
1369 ValueData[ActualNumValueData].Value =
Value->getZExtValue();
1370 ValueData[ActualNumValueData].Count = CntValue;
1371 ActualNumValueData++;
1376std::unique_ptr<InstrProfValueData[]>
1379 uint64_t &TotalC,
bool GetNoICPValue) {
1383 auto ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumValueData);
1385 ActualNumValueData, TotalC, GetNoICPValue))
1387 return ValueDataArray;
1393 bool GetNoICPValue) {
1407 ValueData.
reserve((NOps - 3) / 2);
1408 for (
unsigned I = 3;
I < NOps;
I += 2) {
1409 if (ValueData.
size() >= MaxNumValueData)
1413 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1414 if (!
Value || !Count) {
1421 InstrProfValueData V;
1422 V.Value =
Value->getZExtValue();
1485 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1486 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1491 if (IRInstrVar->isDeclaration())
1495 if (!IRInstrVar->hasInitializer())
1498 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1501 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1506 if (
F.getName().empty())
1512 if (CheckAddressTaken &&
F.hasAddressTaken())
1520 if (!
F.hasComdat()) {
1529 if (InstrProfileOutput.
empty())
1535 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1537 Triple TT(M.getTargetTriple());
1538 if (TT.supportsCOMDAT()) {
1540 ProfileNameVar->
setComdat(M.getOrInsertComdat(
1541 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1546 const std::string &TestFilename,
1548 auto getProfileSum = [IsCS](
const std::string &Filename,
1554 if (
Error E = ReaderOrErr.takeError()) {
1557 auto Reader = std::move(ReaderOrErr.get());
1558 Reader->accumulateCounts(Sum, IsCS);
1576 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1586 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1596 const char *EntryName =
1597 (Level ==
ProgramLevel ?
"functions" :
"edge counters");
1599 OS <<
"Profile overlap infomation for base_profile: " << *
BaseFilename
1600 <<
" and test_profile: " << *
TestFilename <<
"\nProgram level:\n";
1602 OS <<
"Function level:\n"
1611 OS <<
" # of " << EntryName
1617 OS <<
" Mismatched count percentage (Edge): "
1620 OS <<
" Percentage of Edge profile only in test_profile: "
1627 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1630 char ProfileKindName[20] = {0};
1632 case IPVK_IndirectCallTarget:
1633 strncpy(ProfileKindName,
"IndirectCall", 19);
1635 case IPVK_MemOPSize:
1636 strncpy(ProfileKindName,
"MemOP", 19);
1638 case IPVK_VTableTarget:
1639 strncpy(ProfileKindName,
"VTable", 19);
1642 snprintf(ProfileKindName, 19,
"VP[%d]",
I);
1645 OS <<
" " << ProfileKindName
1649 OS <<
" Mismatched count percentage (" << ProfileKindName
1652 OS <<
" Percentage of " << ProfileKindName
1653 <<
" profile only in test_profile: "
1655 OS <<
" " << ProfileKindName
1658 <<
" " << ProfileKindName
1664namespace IndexedInstrProf {
1666 using namespace support;
1667 static_assert(std::is_standard_layout_v<Header>,
1668 "Use standard layout for Header for simplicity");
1671 H.Magic = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1677 H.Version = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1678 if (
H.getIndexedProfileVersion() >
1683 "Please update the reader as needed when a new field is added "
1684 "or when indexed profile version gets bumped.");
1687 H.HashType = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1688 H.HashOffset = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1689 if (
H.getIndexedProfileVersion() >= 8)
1691 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1692 if (
H.getIndexedProfileVersion() >= 9)
1694 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1696 if (
H.getIndexedProfileVersion() >= 10)
1697 H.TemporalProfTracesOffset =
1698 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1699 if (
H.getIndexedProfileVersion() >= 12)
1700 H.VTableNamesOffset =
1701 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1716 "Please update the size computation below if a new field has "
1717 "been added to the header; for a version bump without new "
1718 "fields, add a case statement to fall through to the latest version.");
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
void setComdat(Comdat *C)
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
static bool isLocalLinkage(LinkageTypes Linkage)
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Module * getParent()
Get the module that this global value is contained inside of...
bool isDiscardableIfUnused() const
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ ExternalWeakLinkage
ExternalWeak linkage description.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
std::string message() const override
Return the error message as a string.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Error addSymbolName(StringRef SymbolName)
uint64_t getVTableHashFromAddress(uint64_t Address)
Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
Error addVTableName(StringRef VTableName)
Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
The method name is kept since there are many callers.
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)
Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
ValT lookup(KeyT x, ValT NotFound=ValT()) const
lookup - Return the mapped value at x or NotFound.
This is an important class for using LLVM in a threaded context.
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
MDString * createString(StringRef Str)
Return the given string as metadata.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
const std::string & getSourceFileName() const
Get the module's original source file name.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string str() const
str - Get the contents as an std::string.
const unsigned char * bytes_end() const
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
static constexpr size_t npos
const unsigned char * bytes_begin() const
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
StringRef getPGOFuncNameMetadataName()
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
std::unique_ptr< InstrProfValueData[]> getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
void createPGONameMetadata(GlobalObject &GO, StringRef PGOName)
Create the PGOName metadata if a global object's PGO name is different from its mangled name.
std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
MDNode * mayHaveValueProfileOfKind(const Instruction &Inst, InstrProfValueKind ValueKind)
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
constexpr char kGlobalIdentifierDelimiter
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata)
@ value_site_count_mismatch
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
const std::error_category & instrprof_category()
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
static StringRef getStrippedSourceFileName(const GlobalObject &GO)
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
OutputIt copy(R &&Range, OutputIt Out)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
static bool getValueProfDataFromInstImpl(const MDNode *const MD, const uint32_t MaxNumDataWant, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue)
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
static ValueProfRecordClosure InstrProfRecordClosure
static Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)
NameStrings is a string composed of one of more possibly encoded sub-strings.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
static std::optional< std::string > lookupPGONameFromMetadata(MDNode *MD)
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getPGONameMetadataName()
void consumeError(Error Err)
Consume a Error without doing anything.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName)
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
std::array< double, IPVK_Last - IPVK_First+1 > ValueCounts
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
std::vector< uint64_t > Counts
ArrayRef< InstrProfValueData > getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const
Return the array of profiled values at Site.
CountPseudoKind getCountPseudoKind() const
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
void setPseudoCount(CountPseudoKind Kind)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
std::vector< uint8_t > BitmapBytes
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
void sortByTargetValues()
Sort ValueData ascending by Value.
std::vector< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
CountSumOrPercent Mismatch
static void createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces, std::vector< BPFunctionNode > &Nodes, bool RemoveOutlierUNs=true)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...