20#include "llvm/Config/config.h"
54#include <system_error>
61#define DEBUG_TYPE "instrprof"
65 cl::desc(
"Use full module build paths in the profile counter names for "
66 "static functions."));
78 cl::desc(
"Strip specified level of directory name from source path in "
79 "the profile counter name for static functions."));
82 const std::string &ErrMsg =
"") {
87 case instrprof_error::success:
90 case instrprof_error::eof:
93 case instrprof_error::unrecognized_format:
94 OS <<
"unrecognized instrumentation profile encoding format";
96 case instrprof_error::bad_magic:
97 OS <<
"invalid instrumentation profile data (bad magic)";
99 case instrprof_error::bad_header:
100 OS <<
"invalid instrumentation profile data (file header is corrupt)";
102 case instrprof_error::unsupported_version:
103 OS <<
"unsupported instrumentation profile format version";
105 case instrprof_error::unsupported_hash_type:
106 OS <<
"unsupported instrumentation profile hash type";
108 case instrprof_error::too_large:
109 OS <<
"too much profile data";
111 case instrprof_error::truncated:
112 OS <<
"truncated profile data";
114 case instrprof_error::malformed:
115 OS <<
"malformed instrumentation profile data";
117 case instrprof_error::missing_correlation_info:
118 OS <<
"debug info/binary for correlation is required";
120 case instrprof_error::unexpected_correlation_info:
121 OS <<
"debug info/binary for correlation is not necessary";
123 case instrprof_error::unable_to_correlate_profile:
124 OS <<
"unable to correlate profile";
126 case instrprof_error::invalid_prof:
127 OS <<
"invalid profile created. Please file a bug "
128 "at: " BUG_REPORT_URL
129 " and include the profraw files that caused this error.";
131 case instrprof_error::unknown_function:
132 OS <<
"no profile data available for function";
134 case instrprof_error::hash_mismatch:
135 OS <<
"function control flow change detected (hash mismatch)";
137 case instrprof_error::count_mismatch:
138 OS <<
"function basic block count change detected (counter mismatch)";
140 case instrprof_error::bitmap_mismatch:
141 OS <<
"function bitmap size change detected (bitmap size mismatch)";
143 case instrprof_error::counter_overflow:
144 OS <<
"counter overflow";
146 case instrprof_error::value_site_count_mismatch:
147 OS <<
"function value site count change detected (counter mismatch)";
149 case instrprof_error::compress_failed:
150 OS <<
"failed to compress data (zlib)";
152 case instrprof_error::uncompress_failed:
153 OS <<
"failed to uncompress data (zlib)";
155 case instrprof_error::empty_raw_profile:
156 OS <<
"empty raw profile file";
158 case instrprof_error::zlib_unavailable:
159 OS <<
"profile uses zlib compression but the profile reader was built "
160 "without zlib support";
162 case instrprof_error::raw_profile_version_mismatch:
163 OS <<
"raw profile version mismatch";
165 case instrprof_error::counter_value_too_large:
166 OS <<
"excessively large counter value suggests corrupted profile data";
172 OS <<
": " << ErrMsg;
182class InstrProfErrorCategoryType :
public std::error_category {
183 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
185 std::string message(
int IE)
const override {
193 static InstrProfErrorCategoryType ErrorCategory;
194 return ErrorCategory;
199const char *InstrProfSectNameCommon[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
205const char *InstrProfSectNameCoff[] = {
206#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
211const char *InstrProfSectNamePrefix[] = {
212#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
222 "enable-name-compression",
226 "enable-vtable-value-profiling",
cl::init(
false),
227 cl::desc(
"If true, the virtual table address will be instrumented to know "
228 "the types of a C++ pointer. The information is used in indirect "
229 "call promotion to do selective vtable-based comparison."));
233 bool AddSegmentInfo) {
234 std::string SectName;
237 SectName = InstrProfSectNamePrefix[IPSK];
240 SectName += InstrProfSectNameCoff[IPSK];
242 SectName += InstrProfSectNameCommon[IPSK];
244 if (OF ==
Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
245 SectName +=
",regular,live_support";
265 std::string NewName = std::string(
Name);
271 if (FileName.
empty())
272 NewName = NewName.insert(0,
"<unknown>:");
274 NewName = NewName.insert(0, FileName.
str() +
":");
285 for (
const auto &CI : PathNameStr) {
294 return PathNameStr.
substr(LastPos);
345 MDNode *PGONameMetadata) {
353 return *IRPGOFuncName;
400 if (MangledName.empty())
401 return std::make_pair(
StringRef(), IRPGOName);
402 return std::make_pair(FileName, MangledName);
406 if (FileName.
empty())
426 const char InvalidChars[] =
"-:;<>/\"'";
427 size_t found = VarName.find_first_of(InvalidChars);
428 while (found != std::string::npos) {
429 VarName[found] =
'_';
430 found = VarName.find_first_of(InvalidChars, found + 1);
481 if (!
G.hasName() || !
G.hasMetadata(LLVMContext::MD_type))
483 if (
Error E = addVTableWithName(
499 bool Inserted =
true;
500 std::tie(std::ignore, Inserted) =
506 if (
Error E = mapName(VTablePGOName))
509 StringRef CanonicalName = getCanonicalName(VTablePGOName);
510 if (CanonicalName != VTablePGOName)
511 return mapName(CanonicalName);
523 const uint8_t *EndP = NameStrings.
bytes_end();
530 bool isCompressed = (CompressedSize != 0);
538 UncompressedNameStrings,
544 NameStrings =
toStringRef(UncompressedNameStrings);
547 StringRef(
reinterpret_cast<const char *
>(
P), UncompressedSize);
548 P += UncompressedSize;
557 while (
P < EndP && *
P == 0)
573 this, std::placeholders::_1)))
584 CompressedVTableStrings,
598 const std::string UniqSuffix =
".__uniq.";
599 size_t pos = PGOName.
find(UniqSuffix);
601 pos += UniqSuffix.length();
607 pos = PGOName.
find(
'.', pos);
609 return PGOName.
substr(0, pos);
621 if (
Error E = mapName(PGOFuncName))
624 StringRef CanonicalFuncName = getCanonicalName(PGOFuncName);
625 if (CanonicalFuncName != PGOFuncName)
626 return mapName(CanonicalFuncName);
634 return VTableAddrMap.
lookup(Address, 0);
639 auto It =
partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t>
A) {
640 return A.first < Address;
646 if (It != AddrToMD5Map.end() && It->first == Address)
659 bool doCompression, std::string &Result) {
660 assert(!NameStrs.
empty() &&
"No name data to emit");
662 uint8_t Header[20], *
P = Header;
663 std::string UncompressedNameStrings =
668 "PGO name is invalid (contains separator token)");
670 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
673 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
676 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
677 unsigned HeaderLen =
P - &Header[0];
678 Result.append(HeaderStr, HeaderLen);
683 if (!doCompression) {
684 return WriteStringToResult(0, UncompressedNameStrings);
689 CompressedNameStrings,
692 return WriteStringToResult(CompressedNameStrings.
size(),
699 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
704 std::string &Result,
bool doCompression) {
705 std::vector<std::string> NameStrs;
706 for (
auto *NameVar : NameVars) {
714 std::string &Result,
bool doCompression) {
715 std::vector<std::string> VTableNameStrs;
716 for (
auto *VTable : VTables)
717 VTableNameStrs.push_back(
getPGOName(*VTable));
730 for (
uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
733 for (
size_t I = 0;
I < NumValueSites; ++
I) {
747 double Score = 0.0f, FuncLevelScore = 0.0f;
752 while (
I != IE && J != JE) {
753 if (
I->Value == J->Value) {
761 }
else if (
I->Value < J->Value) {
777 assert(ThisNumValueSites ==
Other.getNumValueSites(ValueKind));
778 if (!ThisNumValueSites)
781 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
782 getOrCreateValueSitesForKind(ValueKind);
784 Other.getValueSitesForKind(ValueKind);
786 ThisSiteRecords[
I].
overlap(OtherSiteRecords[
I], ValueKind, Overlap,
796 bool Mismatch = (
Counts.size() !=
Other.Counts.size());
800 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
803 if (ThisNumValueSites != OtherNumValueSites) {
815 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
821 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
824 MaxCount = std::max(
Other.Counts[
I], MaxCount);
829 if (MaxCount >= ValueCutoff) {
830 double FuncScore = 0.0;
831 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I)
837 FuncLevelOverlap.
Valid =
true;
848 std::vector<InstrProfValueData> Merged;
850 for (
const InstrProfValueData &J : Input.
ValueData) {
851 while (
I != IE &&
I->Value < J.Value) {
852 Merged.push_back(*
I);
855 if (
I != IE &&
I->Value == J.Value) {
860 Merged.push_back(*
I);
866 Merged.insert(Merged.end(),
I, IE);
882void InstrProfRecord::mergeValueProfData(
886 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
887 if (ThisNumValueSites != OtherNumValueSites) {
891 if (!ThisNumValueSites)
893 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
894 getOrCreateValueSitesForKind(ValueKind);
896 Src.getValueSitesForKind(ValueKind);
898 ThisSiteRecords[
I].
merge(OtherSiteRecords[
I], Weight, Warn);
928 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
949 for (
size_t I = 0, E =
Other.BitmapBytes.size();
I < E; ++
I) {
953 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
954 mergeValueProfData(Kind,
Other, Weight, Warn);
957void InstrProfRecord::scaleValueProfData(
960 for (
auto &R : getValueSitesForKind(ValueKind))
966 assert(
D != 0 &&
"D cannot be 0");
967 for (
auto &Count : this->
Counts) {
977 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
978 scaleValueProfData(Kind,
N,
D, Warn);
987 if (ValueKind == IPVK_IndirectCallTarget)
990 if (ValueKind == IPVK_VTableTarget)
1002 std::vector<InstrProfValueSiteRecord> &ValueSites =
1003 getOrCreateValueSitesForKind(ValueKind);
1004 assert(ValueSites.size() == Site);
1006 ValueSites.emplace_back();
1008 ValueSites.emplace_back(VData, VData +
N);
1013 bool RemoveOutlierUNs) {
1016 UtilityNodeT MaxUN = 0;
1022 for (
auto &
Trace : Traces) {
1023 size_t CutoffTimestamp = 1;
1029 It->getSecond() = std::min<size_t>(It->getSecond(),
Timestamp);
1036 for (
auto &[Id, FirstUN] : IdToFirstUN)
1037 for (
auto UN = FirstUN; UN <= MaxUN; ++UN)
1038 IdToUNs[Id].push_back(UN);
1040 IdToFirstUN.
clear();
1043 if (RemoveOutlierUNs) {
1045 for (
auto &[Id, UNs] : IdToUNs)
1046 for (
auto &UN : UNs)
1050 for (
auto &[Id, UNs] : IdToUNs)
1052 return UNFrequency[UN] <= 1 || 2 * UNFrequency[UN] > IdToUNs.
size();
1056 for (
auto &[Id, UNs] : IdToUNs)
1057 Nodes.emplace_back(Id, UNs);
1062 return std::make_pair(IdToFirstTimestamp[L.Id], L.Id) <
1063 std::make_pair(IdToFirstTimestamp[R.Id], R.Id);
1067#define INSTR_PROF_COMMON_API_IMPL
1081 ->getNumValueSites(VKind);
1086 ->getNumValueData(VKind);
1098 llvm::copy(IPR->getValueArrayForSite(K, S), Dst);
1103 (ValueProfData *)(
new (::operator
new(TotalSizeInBytes)) ValueProfData());
1104 memset(VD, 0, TotalSizeInBytes);
1121 Closure.Record = &
Record;
1122 return getValueProfDataSize(&Closure);
1126std::unique_ptr<ValueProfData>
1130 std::unique_ptr<ValueProfData> VPD(
1137 Record.reserveSites(Kind, NumValueSites);
1139 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
1140 for (
uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1141 uint8_t ValueDataCount = this->SiteCountArray[VSite];
1142 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
1143 ValueData += ValueDataCount;
1151 using namespace support;
1157 sys::swapByteOrder<uint32_t>(NumValueSites);
1158 sys::swapByteOrder<uint32_t>(Kind);
1160 uint32_t ND = getValueProfRecordNumValueData(
this);
1161 InstrProfValueData *VD = getValueProfRecordValueData(
this);
1165 sys::swapByteOrder<uint64_t>(VD[
I].
Value);
1166 sys::swapByteOrder<uint64_t>(VD[
I].Count);
1169 sys::swapByteOrder<uint32_t>(NumValueSites);
1170 sys::swapByteOrder<uint32_t>(Kind);
1176 if (NumValueKinds == 0)
1179 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1180 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1181 VR->deserializeTo(
Record, SymTab);
1182 VR = getValueProfRecordNext(VR);
1187 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
1191Error ValueProfData::checkIntegrity() {
1192 if (NumValueKinds > IPVK_Last + 1)
1193 return make_error<InstrProfError>(
1197 return make_error<InstrProfError>(
1200 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1201 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
1202 if (VR->Kind > IPVK_Last)
1204 "value kind is invalid");
1205 VR = getValueProfRecordNext(VR);
1206 if ((
char *)VR - (
char *)
this > (
ptrdiff_t)TotalSize)
1207 return make_error<InstrProfError>(
1209 "value profile address is greater than total size");
1215ValueProfData::getValueProfData(
const unsigned char *
D,
1216 const unsigned char *
const BufferEnd,
1218 using namespace support;
1220 if (
D +
sizeof(ValueProfData) > BufferEnd)
1223 const unsigned char *Header =
D;
1224 uint32_t TotalSize = endian::readNext<uint32_t>(Header, Endianness);
1226 if (
D + TotalSize > BufferEnd)
1230 memcpy(VPD.get(),
D, TotalSize);
1232 VPD->swapBytesToHost(Endianness);
1234 Error E = VPD->checkIntegrity();
1236 return std::move(E);
1238 return std::move(VPD);
1242 using namespace support;
1247 sys::swapByteOrder<uint32_t>(TotalSize);
1248 sys::swapByteOrder<uint32_t>(NumValueKinds);
1250 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1251 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1253 VR = getValueProfRecordNext(VR);
1258 using namespace support;
1263 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1264 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1265 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1269 sys::swapByteOrder<uint32_t>(TotalSize);
1270 sys::swapByteOrder<uint32_t>(NumValueKinds);
1281 for (
const InstrProfValueData &V : VDs)
1306 for (
const auto &VD : VDs) {
1327 if (!
Tag ||
Tag->getString() !=
"VP")
1342 InstrProfValueData ValueData[],
1344 uint64_t &TotalC,
bool GetNoICPValue) {
1351 ActualNumValueData = 0;
1353 for (
unsigned I = 3;
I < NOps;
I += 2) {
1354 if (ActualNumValueData >= MaxNumDataWant)
1358 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1359 if (!
Value || !Count)
1364 ValueData[ActualNumValueData].Value =
Value->getZExtValue();
1365 ValueData[ActualNumValueData].Count = CntValue;
1366 ActualNumValueData++;
1371std::unique_ptr<InstrProfValueData[]>
1374 uint64_t &TotalC,
bool GetNoICPValue) {
1378 auto ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumValueData);
1380 ActualNumValueData, TotalC, GetNoICPValue))
1382 return ValueDataArray;
1388 bool GetNoICPValue) {
1402 ValueData.
reserve((NOps - 3) / 2);
1403 for (
unsigned I = 3;
I < NOps;
I += 2) {
1404 if (ValueData.
size() >= MaxNumValueData)
1408 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1409 if (!
Value || !Count) {
1416 InstrProfValueData V;
1417 V.Value =
Value->getZExtValue();
1430 if (PGOFuncName ==
F.getName())
1468 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1469 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1474 if (IRInstrVar->isDeclaration())
1478 if (!IRInstrVar->hasInitializer())
1481 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1484 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1489 if (
F.getName().empty())
1495 if (CheckAddressTaken &&
F.hasAddressTaken())
1503 if (!
F.hasComdat()) {
1512 if (InstrProfileOutput.
empty())
1518 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1520 Triple TT(M.getTargetTriple());
1521 if (TT.supportsCOMDAT()) {
1523 ProfileNameVar->
setComdat(M.getOrInsertComdat(
1524 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1529 const std::string &TestFilename,
1531 auto getProfileSum = [IsCS](
const std::string &Filename,
1537 if (
Error E = ReaderOrErr.takeError()) {
1540 auto Reader = std::move(ReaderOrErr.get());
1541 Reader->accumulateCounts(Sum, IsCS);
1559 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1569 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1579 const char *EntryName =
1580 (Level ==
ProgramLevel ?
"functions" :
"edge counters");
1582 OS <<
"Profile overlap infomation for base_profile: " << *
BaseFilename
1583 <<
" and test_profile: " << *
TestFilename <<
"\nProgram level:\n";
1585 OS <<
"Function level:\n"
1594 OS <<
" # of " << EntryName
1600 OS <<
" Mismatched count percentage (Edge): "
1603 OS <<
" Percentage of Edge profile only in test_profile: "
1610 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1613 char ProfileKindName[20] = {0};
1615 case IPVK_IndirectCallTarget:
1616 strncpy(ProfileKindName,
"IndirectCall", 19);
1618 case IPVK_MemOPSize:
1619 strncpy(ProfileKindName,
"MemOP", 19);
1621 case IPVK_VTableTarget:
1622 strncpy(ProfileKindName,
"VTable", 19);
1625 snprintf(ProfileKindName, 19,
"VP[%d]",
I);
1628 OS <<
" " << ProfileKindName
1632 OS <<
" Mismatched count percentage (" << ProfileKindName
1635 OS <<
" Percentage of " << ProfileKindName
1636 <<
" profile only in test_profile: "
1638 OS <<
" " << ProfileKindName
1641 <<
" " << ProfileKindName
1647namespace IndexedInstrProf {
1649 using namespace support;
1650 static_assert(std::is_standard_layout_v<Header>,
1651 "Use standard layout for Header for simplicity");
1654 H.Magic = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1660 H.Version = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1661 if (
H.getIndexedProfileVersion() >
1666 "Please update the reader as needed when a new field is added "
1667 "or when indexed profile version gets bumped.");
1670 H.HashType = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1671 H.HashOffset = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1672 if (
H.getIndexedProfileVersion() >= 8)
1674 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1675 if (
H.getIndexedProfileVersion() >= 9)
1677 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1679 if (
H.getIndexedProfileVersion() >= 10)
1680 H.TemporalProfTracesOffset =
1681 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1682 if (
H.getIndexedProfileVersion() >= 12)
1683 H.VTableNamesOffset =
1684 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1699 "Please update the size computation below if a new field has "
1700 "been added to the header; for a version bump without new "
1701 "fields, add a case statement to fall through to the latest version.");
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
void setComdat(Comdat *C)
static bool isLocalLinkage(LinkageTypes Linkage)
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Module * getParent()
Get the module that this global value is contained inside of...
bool isDiscardableIfUnused() const
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ ExternalWeakLinkage
ExternalWeak linkage description.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
std::string message() const override
Return the error message as a string.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Error addSymbolName(StringRef SymbolName)
uint64_t getVTableHashFromAddress(uint64_t Address)
Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
Error addVTableName(StringRef VTableName)
Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
The method name is kept since there are many callers.
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)
Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
ValT lookup(KeyT x, ValT NotFound=ValT()) const
lookup - Return the mapped value at x or NotFound.
This is an important class for using LLVM in a threaded context.
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
MDString * createString(StringRef Str)
Return the given string as metadata.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
const std::string & getSourceFileName() const
Get the module's original source file name.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string str() const
str - Get the contents as an std::string.
const unsigned char * bytes_end() const
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
static constexpr size_t npos
const unsigned char * bytes_begin() const
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
StringRef getPGOFuncNameMetadataName()
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
std::unique_ptr< InstrProfValueData[]> getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
MDNode * mayHaveValueProfileOfKind(const Instruction &Inst, InstrProfValueKind ValueKind)
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
constexpr char kGlobalIdentifierDelimiter
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata)
@ value_site_count_mismatch
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
const std::error_category & instrprof_category()
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
static StringRef getStrippedSourceFileName(const GlobalObject &GO)
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
OutputIt copy(R &&Range, OutputIt Out)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
static bool getValueProfDataFromInstImpl(const MDNode *const MD, const uint32_t MaxNumDataWant, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue)
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
static ValueProfRecordClosure InstrProfRecordClosure
static Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)
NameStrings is a string composed of one of more possibly encoded sub-strings.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
static std::optional< std::string > lookupPGONameFromMetadata(MDNode *MD)
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
void consumeError(Error Err)
Consume a Error without doing anything.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName)
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
std::array< double, IPVK_Last - IPVK_First+1 > ValueCounts
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
std::vector< uint64_t > Counts
ArrayRef< InstrProfValueData > getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const
Return the array of profiled values at Site.
CountPseudoKind getCountPseudoKind() const
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
void setPseudoCount(CountPseudoKind Kind)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
std::vector< uint8_t > BitmapBytes
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
void sortByTargetValues()
Sort ValueData ascending by Value.
std::vector< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
CountSumOrPercent Mismatch
static void createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces, std::vector< BPFunctionNode > &Nodes, bool RemoveOutlierUNs=true)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...