15#include "llvm/Config/llvm-config.h"
25#include <system_error>
28using namespace sampleprof;
32 cl::desc(
"Cutoff value about how many symbols in profile symbol list "
33 "will be used. This is very useful for performance debugging"));
36 "generate-merged-base-profiles",
37 cl::desc(
"When generating nested context-sensitive profiles, always "
38 "generate extra base profile for function with all its context "
39 "profiles merged into it."));
57class SampleProfErrorCategoryType :
public std::error_category {
58 const char *
name()
const noexcept
override {
return "llvm.sampleprof"; }
60 std::string message(
int IE)
const override {
63 case sampleprof_error::success:
65 case sampleprof_error::bad_magic:
66 return "Invalid sample profile data (bad magic)";
67 case sampleprof_error::unsupported_version:
68 return "Unsupported sample profile format version";
69 case sampleprof_error::too_large:
70 return "Too much profile data";
71 case sampleprof_error::truncated:
72 return "Truncated profile data";
73 case sampleprof_error::malformed:
74 return "Malformed sample profile data";
75 case sampleprof_error::unrecognized_format:
76 return "Unrecognized sample profile encoding format";
77 case sampleprof_error::unsupported_writing_format:
78 return "Profile encoding format unsupported for writing operations";
79 case sampleprof_error::truncated_name_table:
80 return "Truncated function name table";
81 case sampleprof_error::not_implemented:
82 return "Unimplemented feature";
83 case sampleprof_error::counter_overflow:
84 return "Counter overflow";
85 case sampleprof_error::ostream_seek_unsupported:
86 return "Ostream does not support seek";
87 case sampleprof_error::uncompress_failed:
88 return "Uncompress failure";
89 case sampleprof_error::zlib_unavailable:
90 return "Zlib is unavailable";
91 case sampleprof_error::hash_mismatch:
92 return "Function hash mismatch";
101 static SampleProfErrorCategoryType ErrorCategory;
102 return ErrorCategory;
123 for (
const auto &
I :
Other.getCallTargets()) {
129#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
139 OS <<
" " <<
I.first <<
":" <<
I.second;
144#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
159 OS << TotalSamples <<
", " << TotalHeadSamples <<
", " << BodySamples.size()
160 <<
" sampled lines\n";
163 if (!BodySamples.empty()) {
164 OS <<
"Samples collected in the function's body {\n";
166 for (
const auto &SI : SortedBodySamples.
get()) {
168 OS << SI->first <<
": " << SI->second;
173 OS <<
"No samples collected in the function's body\n";
177 if (!CallsiteSamples.empty()) {
178 OS <<
"Samples collected in inlined callsites {\n";
181 for (
const auto &CS : SortedCallsiteSamples.
get()) {
182 for (
const auto &FS : CS->second) {
184 OS << CS->first <<
": inlined callee: " << FS.second.getFunction()
186 FS.second.print(
OS, Indent + 4);
192 OS <<
"No inlined callsites in this function\n";
204 std::vector<NameFunctionSamples> &SortedProfiles) {
205 for (
const auto &
I : ProfileMap) {
206 SortedProfiles.push_back(std::make_pair(
I.first, &
I.second));
210 if (
A.second->getTotalSamples() ==
B.second->getTotalSamples())
211 return A.second->getContext() <
B.second->getContext();
212 return A.second->getTotalSamples() >
B.second->getTotalSamples();
217 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &
229 DIL->getDiscriminator()),
232 unsigned Discriminator =
244 for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
246 StringRef Name = PrevDIL->getScope()->getSubprogram()->getLinkageName();
248 Name = PrevDIL->getScope()->getSubprogram()->getName();
258 for (
int i = S.
size() - 1; i >= 0 && FS !=
nullptr; i--) {
259 FS = FS->findFunctionSamplesAt(S[i].first, S[i].second, Remapper);
266 for (
const auto &BS : BodySamples)
267 for (
const auto &TS : BS.second.getCallTargets())
270 for (
const auto &CS : CallsiteSamples) {
271 for (
const auto &NameFS : CS.second) {
272 NameSet.
insert(NameFS.first);
273 NameFS.second.findAllNames(NameSet);
284 if (iter == CallsiteSamples.end())
287 if (FS != iter->second.end())
292 if (FS != iter->second.end())
299 if (!CalleeName.
empty())
303 for (
const auto &NameFS : iter->second)
304 if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {
305 MaxTotalSamples = NameFS.second.getTotalSamples();
311#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
317 const char *ListStart =
reinterpret_cast<const char *
>(
Data);
323 Size += Str.size() + 1;
333 uint32_t ColdContextFrameLength,
bool TrimBaseProfileOnly) {
334 if (!TrimColdContext && !MergeColdContext)
344 if (MergeColdContext)
345 TrimBaseProfileOnly =
false;
349 std::vector<std::pair<hash_code, const FunctionSamples *>> ColdProfiles;
350 for (
const auto &
I : ProfileMap) {
354 (!TrimBaseProfileOnly ||
Context.isBaseContext()))
355 ColdProfiles.emplace_back(
I.first, &
I.second);
361 for (
const auto &
I : ColdProfiles) {
362 if (MergeColdContext) {
368 MergedProfile.
merge(*
I.second);
370 ProfileMap.erase(
I.first);
374 for (
const auto &
I : MergedProfileMap) {
377 ProfileMap.find(
I.second.getContext()) == ProfileMap.end())
384 OrigProfile.
merge(
I.second);
391 std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
394 std::string OutputString;
395 for (
auto &
Sym : SortedList) {
396 OutputString.append(
Sym.str());
397 OutputString.append(1,
'\0');
405 OS <<
"======== Dump profile symbol list ========\n";
406 std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
409 for (
auto &
Sym : SortedList)
419 assert(It->second.FuncName == CalleeName &&
420 "Hash collision for child context node");
429 : ProfileMap(Profiles) {
430 for (
auto &FuncSample : Profiles) {
432 auto *NewNode = getOrCreateContextPath(FSamples->
getContext());
433 assert(!NewNode->FuncSamples &&
"New node cannot have sample profile");
434 NewNode->FuncSamples = FSamples;
439ProfileConverter::getOrCreateContextPath(
const SampleContext &Context) {
440 auto Node = &RootFrame;
442 for (
auto &Callsite :
Context.getContextFrames()) {
443 Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.Func);
444 CallSiteLoc = Callsite.Location;
453 auto *NodeProfile =
Node.FuncSamples;
454 for (
auto &It :
Node.AllChildFrames) {
455 auto &ChildNode = It.second;
457 auto *ChildProfile = ChildNode.FuncSamples;
463 ChildProfile->getContext().setFunction(OrigChildContext.
getFunction());
466 auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
467 SamplesMap.emplace(OrigChildContext.
getFunction(), *ChildProfile);
468 NodeProfile->addTotalSamples(ChildProfile->getTotalSamples());
471 auto Count = NodeProfile->removeCalledTargetAndBodySample(
472 ChildNode.CallSiteLoc.LineOffset, ChildNode.CallSiteLoc.Discriminator,
474 NodeProfile->removeTotalSamples(Count);
485 ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
486 NewChildProfileHash = ChildProfile->getContext().getHashCode();
488 ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
489 NewChildProfileHash = ChildProfile->getContext().getHashCode();
490 auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
491 SamplesMap[ChildProfile->getFunction()].getContext().setAttribute(
498 if (NewChildProfileHash != OrigChildContextHash)
499 ProfileMap.
erase(OrigChildContextHash);
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > GenerateMergedBaseProfiles("generate-merged-base-profiles", cl::desc("When generating nested context-sensitive profiles, always " "generate extra base profile for function with all its context " "profiles merged into it."))
static cl::opt< uint64_t > ProfileSymbolListCutOff("profile-symbol-list-cutoff", cl::Hidden, cl::init(-1), cl::desc("Cutoff value about how many symbols in profile symbol list " "will be used. This is very useful for performance debugging"))
unsigned getBaseDiscriminator() const
Returns the base discriminator stored in the discriminator.
Implements a dense probed hash-table based set.
reference emplace_back(ArgTypes &&... Args)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
This class represents a function that is read from a sample profile.
Representation of the samples collected for a function.
static bool ProfileIsPreInlined
static uint64_t getCallSiteHash(FunctionId Callee, const LineLocation &Callsite)
Returns a unique hash code for a combination of a callsite location and the callee function name.
const LineLocation & mapIRLocToProfileLoc(const LineLocation &IRLoc) const
FunctionId getFunction() const
Return the function name.
uint64_t getFunctionHash() const
const FunctionSamples * findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName, SampleProfileReaderItaniumRemapper *Remapper) const
Returns a pointer to FunctionSamples at the given callsite location Loc with callee CalleeName.
static bool ProfileIsProbeBased
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
void findAllNames(DenseSet< FunctionId > &NameSet) const
static unsigned getOffset(const DILocation *DIL)
Returns the line offset to the start line of the subprogram.
static bool ProfileIsFS
If this profile uses flow sensitive discriminators.
SampleContext & getContext() const
static bool HasUniqSuffix
Whether the profile contains any ".__uniq." suffix in a name.
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
void print(raw_ostream &OS=dbgs(), unsigned Indent=0) const
Print the samples collected for a function on stream OS.
sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight=1)
Merge the samples in Other into this one.
static LineLocation getCallSiteIdentifier(const DILocation *DIL, bool ProfileIsFS=false)
Returns a unique call site identifier for a given debug location of a call instruction.
const FunctionSamples * findFunctionSamples(const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper=nullptr) const
Get the FunctionSamples of the inline instance where DIL originates from.
static bool UseMD5
Whether the profile uses MD5 to represent string.
ProfileConverter(SampleProfileMap &Profiles)
std::error_code write(raw_ostream &OS)
void dump(raw_ostream &OS=dbgs()) const
void add(StringRef Name, bool copy=false)
copy indicates whether we need to copy the underlying memory for the input Name.
std::error_code read(const uint8_t *Data, uint64_t ListSize)
void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext, uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly)
uint64_t getHashCode() const
FunctionId getFunction() const
This class provides operator overloads to the map container using MD5 as the key type,...
mapped_type & Create(const SampleContext &Ctx)
size_t erase(const SampleContext &Ctx)
SampleProfileReaderItaniumRemapper remaps the profile data from a sample profile data reader,...
std::optional< StringRef > lookUpNameInProfile(StringRef FunctionName)
Return the equivalent name in the profile for FunctionName if it exists.
Representation of a single sample record.
bool hasCalls() const
Return true if this sample record contains function calls.
sampleprof_error merge(const SampleRecord &Other, uint64_t Weight=1)
Merge the samples in Other into this record.
sampleprof_error addSamples(uint64_t S, uint64_t Weight=1)
Increment the number of samples for this record by S.
const SortedCallTargetSet getSortedCallTargets() const
void print(raw_ostream &OS, unsigned Indent) const
Print the sample record to the stream OS indented by Indent.
sampleprof_error addCalledTarget(FunctionId F, uint64_t S, uint64_t Weight=1)
Add called function F with samples S.
Sort a LocationT->SampleT map by LocationT.
const SamplesWithLocList & get() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
initializer< Ty > init(const Ty &Val)
static FunctionId getRepInFormat(StringRef Name)
Get the proper representation of a string according to whether the current Format uses MD5 to represe...
void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)
std::pair< hash_code, const FunctionSamples * > NameFunctionSamples
@ ContextDuplicatedIntoBase
raw_ostream & operator<<(raw_ostream &OS, const FunctionId &Obj)
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
const std::error_category & sampleprof_category()
sampleprof_error MergeResult(sampleprof_error &Accumulator, sampleprof_error Result)
static uint32_t extractProbeIndex(uint32_t Value)
Represents the relative location of an instruction.
void print(raw_ostream &OS) const
FrameNode * getOrCreateChildFrame(const LineLocation &CallSite, FunctionId CalleeName)
std::map< uint64_t, FrameNode > AllChildFrames