32 "static-func-full-module-prefix",
cl::init(
false),
33 cl::desc(
"Use full module build paths in the profile counter names for "
34 "static functions."));
44 return "Unrecognized instrumentation profile encoding format";
46 return "Invalid instrumentation profile data (bad magic)";
48 return "Invalid instrumentation profile data (file header is corrupt)";
50 return "Unsupported instrumentation profile format version";
52 return "Unsupported instrumentation profile hash type";
54 return "Too much profile data";
56 return "Truncated profile data";
58 return "Malformed instrumentation profile data";
60 return "No profile data available for function";
62 return "Function control flow change detected (hash mismatch)";
64 return "Function basic block count change detected (counter mismatch)";
66 return "Counter overflow";
68 return "Function value site count change detected (counter mismatch)";
70 return "Failed to compress data (zlib)";
72 return "Failed to uncompress data (zlib)";
74 return "Empty raw profile file";
83 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
84 std::string message(
int IE)
const override {
85 return getInstrProfErrString(static_cast<instrprof_error>(IE));
110 ++NumCountMismatches;
113 ++NumCounterOverflows;
116 ++NumValueSiteCountMismatches;
124 return getInstrProfErrString(Err);
159 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
170 if (FileName.
empty())
189 const char *InvalidChars =
"-:<>/\"'";
190 size_t found = VarName.find_first_of(InvalidChars);
191 while (found != std::string::npos) {
192 VarName[found] =
'_';
193 found = VarName.find_first_of(InvalidChars, found + 1);
245 bool doCompression, std::string &Result) {
246 assert(NameStrs.size() &&
"No name data to emit");
248 uint8_t Header[16], *
P = Header;
249 std::string UncompressedNameStrings =
254 "PGO name is invalid (contains separator token)");
256 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
259 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
262 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
263 unsigned HeaderLen = P - &Header[0];
264 Result.append(HeaderStr, HeaderLen);
269 if (!doCompression) {
270 return WriteStringToResult(0, UncompressedNameStrings);
281 return WriteStringToResult(CompressedNameStrings.
size(),
282 CompressedNameStrings);
288 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
293 std::string &Result,
bool doCompression) {
294 std::vector<std::string> NameStrs;
295 for (
auto *NameVar : NameVars) {
303 const uint8_t *
P =
reinterpret_cast<const uint8_t *
>(NameStrings.
data());
304 const uint8_t *EndP =
reinterpret_cast<const uint8_t *
>(NameStrings.
data() +
312 bool isCompressed = (CompressedSize != 0);
316 StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
323 UncompressedNameStrings.
size());
326 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
327 P += UncompressedSize;
335 while (P < EndP && *P == 0)
351 while (
I !=
IE &&
I->Value < J->Value)
353 if (
I !=
IE &&
I->Value == J->Value) {
377 void InstrProfRecord::mergeValueProfData(
uint32_t ValueKind,
382 if (ThisNumValueSites != OtherNumValueSites) {
386 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
387 getValueSitesForKind(ValueKind);
388 std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
389 Src.getValueSitesForKind(ValueKind);
391 ThisSiteRecords[
I].
merge(
SIPE, OtherSiteRecords[
I], Weight);
402 for (
size_t I = 0,
E = Other.
Counts.size();
I <
E; ++
I) {
411 mergeValueProfData(
Kind, Other, Weight);
414 void InstrProfRecord::scaleValueProfData(
uint32_t ValueKind, uint64_t Weight) {
416 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
417 getValueSitesForKind(ValueKind);
423 for (
auto &Count : this->
Counts) {
430 scaleValueProfData(
Kind, Weight);
434 uint64_t InstrProfRecord::remapValue(uint64_t
Value,
uint32_t ValueKind,
439 case IPVK_IndirectCallTarget: {
441 std::lower_bound(ValueMap->begin(), ValueMap->end(), Value,
442 [](
const std::pair<uint64_t, uint64_t> &LHS,
443 uint64_t RHS) {
return LHS.first < RHS; });
448 if (Result != ValueMap->end() && Result->first == Value)
449 Value = (uint64_t)Result->second;
462 VData[
I].Value = remapValue(VData[
I].Value, ValueKind, ValueMap);
464 std::vector<InstrProfValueSiteRecord> &ValueSites =
465 getValueSitesForKind(ValueKind);
467 ValueSites.emplace_back();
469 ValueSites.emplace_back(VData, VData + N);
472 #define INSTR_PROF_COMMON_API_IMPL
481 return reinterpret_cast<const InstrProfRecord *
>(Record)->getNumValueKinds();
486 ->getNumValueSites(VKind);
491 ->getNumValueData(VKind);
497 ->getNumValueDataForSite(VK, S);
502 reinterpret_cast<const InstrProfRecord *
>(R)->getValueForSite(Dst, K, S);
507 (ValueProfData *)(
new (::
operator new(TotalSizeInBytes)) ValueProfData());
508 memset(VD, 0, TotalSizeInBytes);
529 std::unique_ptr<ValueProfData>
533 std::unique_ptr<ValueProfData> VPD(
542 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
543 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
544 uint8_t ValueDataCount = this->SiteCountArray[VSite];
546 ValueData += ValueDataCount;
555 using namespace support;
560 sys::swapByteOrder<uint32_t>(NumValueSites);
561 sys::swapByteOrder<uint32_t>(
Kind);
563 uint32_t ND = getValueProfRecordNumValueData(
this);
564 InstrProfValueData *VD = getValueProfRecordValueData(
this);
568 sys::swapByteOrder<uint64_t>(VD[
I].Value);
569 sys::swapByteOrder<uint64_t>(VD[
I].Count);
572 sys::swapByteOrder<uint32_t>(NumValueSites);
573 sys::swapByteOrder<uint32_t>(
Kind);
579 if (NumValueKinds == 0)
582 ValueProfRecord *VR = getFirstValueProfRecord(
this);
583 for (
uint32_t K = 0; K < NumValueKinds; K++) {
584 VR->deserializeTo(Record, VMap);
585 VR = getValueProfRecordNext(VR);
591 using namespace support;
593 return endian::readNext<T, little, unaligned>(
D);
595 return endian::readNext<T, big, unaligned>(
D);
599 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
603 Error ValueProfData::checkIntegrity() {
604 if (NumValueKinds > IPVK_Last + 1)
607 if (TotalSize %
sizeof(uint64_t))
610 ValueProfRecord *VR = getFirstValueProfRecord(
this);
611 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
612 if (VR->Kind > IPVK_Last)
614 VR = getValueProfRecordNext(VR);
615 if ((
char *)VR - (
char *)
this > (ptrdiff_t)TotalSize)
622 ValueProfData::getValueProfData(
const unsigned char *
D,
623 const unsigned char *
const BufferEnd,
625 using namespace support;
626 if (D +
sizeof(ValueProfData) > BufferEnd)
629 const unsigned char *Header =
D;
630 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
631 if (D + TotalSize > BufferEnd)
635 memcpy(VPD.get(),
D, TotalSize);
637 VPD->swapBytesToHost(Endianness);
639 Error E = VPD->checkIntegrity();
643 return std::move(VPD);
647 using namespace support;
651 sys::swapByteOrder<uint32_t>(TotalSize);
652 sys::swapByteOrder<uint32_t>(NumValueKinds);
654 ValueProfRecord *VR = getFirstValueProfRecord(
this);
655 for (
uint32_t K = 0; K < NumValueKinds; K++) {
657 VR = getValueProfRecordNext(VR);
662 using namespace support;
666 ValueProfRecord *VR = getFirstValueProfRecord(
this);
667 for (
uint32_t K = 0; K < NumValueKinds; K++) {
668 ValueProfRecord *NVR = getValueProfRecordNext(VR);
672 sys::swapByteOrder<uint32_t>(TotalSize);
673 sys::swapByteOrder<uint32_t>(NumValueKinds);
685 std::unique_ptr<InstrProfValueData[]> VD =
710 for (
auto &VD : VDs) {
724 InstrProfValueData ValueData[],
725 uint32_t &ActualNumValueData, uint64_t &TotalC) {
756 ActualNumValueData = 0;
758 for (
unsigned I = 3;
I < NOps;
I += 2) {
759 if (ActualNumValueData >= MaxNumValueData)
763 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
764 if (!Value || !Count)
767 ValueData[ActualNumValueData].Count = Count->getZExtValue();
768 ActualNumValueData++;
779 if (PGOFuncName == F.
getName())
794 if (!TT.isOSBinFormatELF())
819 if (!IRInstrVar || IRInstrVar->isDeclaration() ||
820 IRInstrVar->hasLocalLinkage())
824 if (!IRInstrVar->hasInitializer())
827 const Constant *InitVal = IRInstrVar->getInitializer();
831 return (dyn_cast<ConstantInt>(InitVal)->getZExtValue() &
832 VARIANT_MASK_IR_PROF) != 0;
void setVisibility(VisibilityTypes V)
void scale(uint64_t Weight)
Scale up profile counts (including value profile data) by Weight.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
void push_back(const T &Elt)
LinkageTypes getLinkage() const
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
DiagnosticInfoOptimizationBase::Argument NV
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
A Module instance is used to store all the information related to an LLVM module. ...
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
static MDString * get(LLVMContext &Context, StringRef Str)
Available for inspection, not emission.
unsigned getNumOperands() const
Return number of MDNode operands.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Like Internal, but omit from symbol table.
Externally visible function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig)
static IntegerType * getInt64Ty(LLVMContext &C)
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
std::enable_if< std::is_unsigned< T >::value, T >::type SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
static bool isLocalLinkage(LinkageTypes Linkage)
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
StringRef getName() const
Return a constant reference to the value's name.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
StringRef getName() const
Get a short "name" for the module.
const std::error_category & instrprof_category()
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Status compress(StringRef InputBuffer, SmallVectorImpl< char > &CompressedBuffer, CompressionLevel Level=DefaultCompression)
void merge(InstrProfRecord &Other, uint64_t Weight=1)
Merge the counts in Other into this one.
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Error collectPGOFuncNameStrings(const std::vector< std::string > &NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Tagged union holding either a T or a Error.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name...
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
void addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
bool isDiscardableIfUnused() const
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr)
Utility function to decode a ULEB128 value.
void merge(SoftInstrProfErrors &SIPE, InstrProfValueSiteRecord &Input, uint64_t Weight=1)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
static ManagedStatic< InstrProfErrorCategoryType > ErrorCategory
std::string message() const override
Return the error message as a string.
Status uncompress(StringRef InputBuffer, char *UncompressedBuffer, size_t &UncompressedSize)
ExternalWeak linkage description.
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
StringRef filename(StringRef path)
Get filename.
Same, but only replaced by something equivalent.
initializer< Ty > init(const Ty &Val)
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC)
Extract the value profile data from Inst which is annotated with value profile meta data...
void scale(SoftInstrProfErrors &SIPE, uint64_t Weight)
Scale up value profile data counts.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
This is an important class for using LLVM in a threaded context.
This is an important base class in LLVM.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static ValueProfRecordClosure InstrProfRecordClosure
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
static ManagedStatic< _object_error_category > error_category
#define LLVM_ATTRIBUTE_UNUSED
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(false), cl::desc("Use full module build paths in the profile counter names for ""static functions."))
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Value(Type *Ty, unsigned scid)
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Triple - Helper class for working with autoconf configuration names.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, ValueMapType *ValueMap)
Add ValueData for ValueKind at value Site.
std::enable_if< std::is_unsigned< T >::value, T >::type SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
StringRef getString() const
const MDOperand & getOperand(unsigned I) const
static ErrorSuccess success()
Create a success value.
This is the shared class of boolean and integer constants.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Keep one copy of function when linking (inline)
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
GlobalVariable * getNamedGlobal(StringRef Name)
Return the global variable in the module with the specified name, of arbitrary type.
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e...
StringRef getPGOFuncNameMetadataName()
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
LinkageTypes
An enumeration for the kinds of linkage for global values.
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
void sortByTargetValues()
Sort ValueData ascending by Value.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
void setMetadata(unsigned KindID, MDNode *MD)
Set a particular kind of metadata attachment.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
std::vector< uint64_t > Counts
bool hasAddressTaken(const User **=nullptr) const
hasAddressTaken - returns true if there are any uses of this function other than direct calls or invo...
pointer data()
Return a pointer to the vector's buffer, even if empty().
Profiling information for a single function.
static IntegerType * getInt32Ty(LLVMContext &C)
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
MDString * createString(StringRef Str)
Return the given string as metadata.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Rename collisions when linking (static functions).
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=0) const
Return the array of profiled values at Site.
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
Lightweight error class with error context and mandatory checking.
support::endianness getHostEndianness()
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
void encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned Padding=0)
Utility function to encode a ULEB128 value to an output stream.
std::vector< std::pair< uint64_t, uint64_t > > ValueMapType
StringRef - Represent a constant reference to a string, i.e.
ManagedStatic - This transparently changes the behavior of global statics to be lazily constructed on...
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
LLVMContext & getContext() const
Get the global data context.