LLVM 18.0.0git
InstrProf.cpp
Go to the documentation of this file.
1//===- InstrProf.cpp - Instrumented profiling format support --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for clang's instrumentation based PGO and
10// coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Config/config.h"
22#include "llvm/IR/Constant.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalValue.h"
27#include "llvm/IR/Instruction.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/IR/Mangler.h"
31#include "llvm/IR/Metadata.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/Type.h"
39#include "llvm/Support/Endian.h"
40#include "llvm/Support/Error.h"
42#include "llvm/Support/LEB128.h"
44#include "llvm/Support/Path.h"
48#include <algorithm>
49#include <cassert>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <memory>
54#include <string>
55#include <system_error>
56#include <type_traits>
57#include <utility>
58#include <vector>
59
60using namespace llvm;
61
63 "static-func-full-module-prefix", cl::init(true), cl::Hidden,
64 cl::desc("Use full module build paths in the profile counter names for "
65 "static functions."));
66
67// This option is tailored to users that have different top-level directory in
68// profile-gen and profile-use compilation. Users need to specific the number
69// of levels to strip. A value larger than the number of directories in the
70// source file will strip all the directory names and only leave the basename.
71//
72// Note current ThinLTO module importing for the indirect-calls assumes
73// the source directory name not being stripped. A non-zero option value here
74// can potentially prevent some inter-module indirect-call-promotions.
76 "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
77 cl::desc("Strip specified level of directory name from source path in "
78 "the profile counter name for static functions."));
79
81 const std::string &ErrMsg = "") {
82 std::string Msg;
84
85 switch (Err) {
86 case instrprof_error::success:
87 OS << "success";
88 break;
89 case instrprof_error::eof:
90 OS << "end of File";
91 break;
92 case instrprof_error::unrecognized_format:
93 OS << "unrecognized instrumentation profile encoding format";
94 break;
95 case instrprof_error::bad_magic:
96 OS << "invalid instrumentation profile data (bad magic)";
97 break;
98 case instrprof_error::bad_header:
99 OS << "invalid instrumentation profile data (file header is corrupt)";
100 break;
101 case instrprof_error::unsupported_version:
102 OS << "unsupported instrumentation profile format version";
103 break;
104 case instrprof_error::unsupported_hash_type:
105 OS << "unsupported instrumentation profile hash type";
106 break;
107 case instrprof_error::too_large:
108 OS << "too much profile data";
109 break;
110 case instrprof_error::truncated:
111 OS << "truncated profile data";
112 break;
113 case instrprof_error::malformed:
114 OS << "malformed instrumentation profile data";
115 break;
116 case instrprof_error::missing_debug_info_for_correlation:
117 OS << "debug info for correlation is required";
118 break;
119 case instrprof_error::unexpected_debug_info_for_correlation:
120 OS << "debug info for correlation is not necessary";
121 break;
122 case instrprof_error::unable_to_correlate_profile:
123 OS << "unable to correlate profile";
124 break;
125 case instrprof_error::invalid_prof:
126 OS << "invalid profile created. Please file a bug "
127 "at: " BUG_REPORT_URL
128 " and include the profraw files that caused this error.";
129 break;
130 case instrprof_error::unknown_function:
131 OS << "no profile data available for function";
132 break;
133 case instrprof_error::hash_mismatch:
134 OS << "function control flow change detected (hash mismatch)";
135 break;
136 case instrprof_error::count_mismatch:
137 OS << "function basic block count change detected (counter mismatch)";
138 break;
139 case instrprof_error::counter_overflow:
140 OS << "counter overflow";
141 break;
142 case instrprof_error::value_site_count_mismatch:
143 OS << "function value site count change detected (counter mismatch)";
144 break;
145 case instrprof_error::compress_failed:
146 OS << "failed to compress data (zlib)";
147 break;
148 case instrprof_error::uncompress_failed:
149 OS << "failed to uncompress data (zlib)";
150 break;
151 case instrprof_error::empty_raw_profile:
152 OS << "empty raw profile file";
153 break;
154 case instrprof_error::zlib_unavailable:
155 OS << "profile uses zlib compression but the profile reader was built "
156 "without zlib support";
157 break;
158 case instrprof_error::raw_profile_version_mismatch:
159 OS << "raw profile version mismatch";
160 break;
161 }
162
163 // If optional error message is not empty, append it to the message.
164 if (!ErrMsg.empty())
165 OS << ": " << ErrMsg;
166
167 return OS.str();
168}
169
170namespace {
171
172// FIXME: This class is only here to support the transition to llvm::Error. It
173// will be removed once this transition is complete. Clients should prefer to
174// deal with the Error value directly, rather than converting to error_code.
175class InstrProfErrorCategoryType : public std::error_category {
176 const char *name() const noexcept override { return "llvm.instrprof"; }
177
178 std::string message(int IE) const override {
179 return getInstrProfErrString(static_cast<instrprof_error>(IE));
180 }
181};
182
183} // end anonymous namespace
184
185const std::error_category &llvm::instrprof_category() {
186 static InstrProfErrorCategoryType ErrorCategory;
187 return ErrorCategory;
188}
189
190namespace {
191
192const char *InstrProfSectNameCommon[] = {
193#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
194 SectNameCommon,
196};
197
198const char *InstrProfSectNameCoff[] = {
199#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
200 SectNameCoff,
202};
203
204const char *InstrProfSectNamePrefix[] = {
205#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
206 Prefix,
208};
209
210} // namespace
211
212namespace llvm {
213
215 "enable-name-compression",
216 cl::desc("Enable name/filename string compression"), cl::init(true));
217
220 bool AddSegmentInfo) {
221 std::string SectName;
222
223 if (OF == Triple::MachO && AddSegmentInfo)
224 SectName = InstrProfSectNamePrefix[IPSK];
225
226 if (OF == Triple::COFF)
227 SectName += InstrProfSectNameCoff[IPSK];
228 else
229 SectName += InstrProfSectNameCommon[IPSK];
230
231 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
232 SectName += ",regular,live_support";
233
234 return SectName;
235}
236
237std::string InstrProfError::message() const {
238 return getInstrProfErrString(Err, Msg);
239}
240
241char InstrProfError::ID = 0;
242
243std::string getPGOFuncName(StringRef RawFuncName,
245 StringRef FileName,
247 return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
248}
249
250// Strip NumPrefix level of directory name from PathNameStr. If the number of
251// directory separators is less than NumPrefix, strip all the directories and
252// leave base file name only.
253static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
254 uint32_t Count = NumPrefix;
255 uint32_t Pos = 0, LastPos = 0;
256 for (auto & CI : PathNameStr) {
257 ++Pos;
259 LastPos = Pos;
260 --Count;
261 }
262 if (Count == 0)
263 break;
264 }
265 return PathNameStr.substr(LastPos);
266}
267
269 StringRef FileName(F.getParent()->getSourceFileName());
270 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
271 if (StripLevel < StaticFuncStripDirNamePrefix)
272 StripLevel = StaticFuncStripDirNamePrefix;
273 if (StripLevel)
274 FileName = stripDirPrefix(FileName, StripLevel);
275 return FileName;
276}
277
278// The PGO name has the format [<filepath>;]<linkage-name> where <filepath>; is
279// provided if linkage is local and <linkage-name> is the mangled function
280// name. The filepath is used to discriminate possibly identical function names.
281// ; is used because it is unlikely to be found in either <filepath> or
282// <linkage-name>.
283//
284// Older compilers used getPGOFuncName() which has the format
285// [<filepath>:]<function-name>. <filepath> is used to discriminate between
286// possibly identical function names when linkage is local and <function-name>
287// simply comes from F.getName(). This caused trouble for Objective-C functions
288// which commonly have :'s in their names. Also, since <function-name> is not
289// mangled, they cannot be passed to Mach-O linkers via -order_file. We still
290// need to compute this name to lookup functions from profiles built by older
291// compilers.
292static std::string getIRPGOFuncName(const Function &F,
294 StringRef FileName) {
297 Name.append(FileName.empty() ? "<unknown>" : FileName);
298 Name.append(";");
299 }
300 Mangler().getNameWithPrefix(Name, &F, /*CannotUsePrivateLabel=*/true);
301 return Name.str().str();
302}
303
304static std::optional<std::string> lookupPGOFuncName(const Function &F) {
305 if (MDNode *MD = getPGOFuncNameMetadata(F)) {
306 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
307 return S.str();
308 }
309 return {};
310}
311
312// See getPGOFuncName()
313std::string getIRPGOFuncName(const Function &F, bool InLTO) {
314 if (!InLTO) {
315 auto FileName = getStrippedSourceFileName(F);
316 return getIRPGOFuncName(F, F.getLinkage(), FileName);
317 }
318
319 // In LTO mode (when InLTO is true), first check if there is a meta data.
320 if (auto IRPGOFuncName = lookupPGOFuncName(F))
321 return *IRPGOFuncName;
322
323 // If there is no meta data, the function must be a global before the value
324 // profile annotation pass. Its current linkage may be internal if it is
325 // internalized in LTO mode.
327}
328
329// Return the PGOFuncName. This function has some special handling when called
330// in LTO optimization. The following only applies when calling in LTO passes
331// (when \c InLTO is true): LTO's internalization privatizes many global linkage
332// symbols. This happens after value profile annotation, but those internal
333// linkage functions should not have a source prefix.
334// Additionally, for ThinLTO mode, exported internal functions are promoted
335// and renamed. We need to ensure that the original internal PGO name is
336// used when computing the GUID that is compared against the profiled GUIDs.
337// To differentiate compiler generated internal symbols from original ones,
338// PGOFuncName meta data are created and attached to the original internal
339// symbols in the value profile annotation step
340// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
341// data, its original linkage must be non-internal.
342std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
343 if (!InLTO) {
344 auto FileName = getStrippedSourceFileName(F);
345 return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
346 }
347
348 // In LTO mode (when InLTO is true), first check if there is a meta data.
349 if (auto PGOFuncName = lookupPGOFuncName(F))
350 return *PGOFuncName;
351
352 // If there is no meta data, the function must be a global before the value
353 // profile annotation pass. Its current linkage may be internal if it is
354 // internalized in LTO mode.
355 return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
356}
357
358// See getIRPGOFuncName() for a discription of the format.
359std::pair<StringRef, StringRef>
361 auto [FileName, FuncName] = IRPGOFuncName.split(';');
362 if (FuncName.empty())
363 return std::make_pair(StringRef(), IRPGOFuncName);
364 return std::make_pair(FileName, FuncName);
365}
366
368 if (FileName.empty())
369 return PGOFuncName;
370 // Drop the file name including ':'. See also getPGOFuncName.
371 if (PGOFuncName.startswith(FileName))
372 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
373 return PGOFuncName;
374}
375
376// \p FuncName is the string used as profile lookup key for the function. A
377// symbol is created to hold the name. Return the legalized symbol name.
378std::string getPGOFuncNameVarName(StringRef FuncName,
380 std::string VarName = std::string(getInstrProfNameVarPrefix());
381 VarName += FuncName;
382
383 if (!GlobalValue::isLocalLinkage(Linkage))
384 return VarName;
385
386 // Now fix up illegal chars in local VarName that may upset the assembler.
387 const char InvalidChars[] = "-:;<>/\"'";
388 size_t found = VarName.find_first_of(InvalidChars);
389 while (found != std::string::npos) {
390 VarName[found] = '_';
391 found = VarName.find_first_of(InvalidChars, found + 1);
392 }
393 return VarName;
394}
395
398 StringRef PGOFuncName) {
399 // We generally want to match the function's linkage, but available_externally
400 // and extern_weak both have the wrong semantics, and anything that doesn't
401 // need to link across compilation units doesn't need to be visible at all.
404 else if (Linkage == GlobalValue::AvailableExternallyLinkage)
406 else if (Linkage == GlobalValue::InternalLinkage ||
409
410 auto *Value =
411 ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
412 auto FuncNameVar =
413 new GlobalVariable(M, Value->getType(), true, Linkage, Value,
414 getPGOFuncNameVarName(PGOFuncName, Linkage));
415
416 // Hide the symbol so that we correctly get a copy for each executable.
417 if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
418 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
419
420 return FuncNameVar;
421}
422
424 return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
425}
426
428 for (Function &F : M) {
429 // Function may not have a name: like using asm("") to overwrite the name.
430 // Ignore in this case.
431 if (!F.hasName())
432 continue;
433 if (Error E = addFuncWithName(F, getIRPGOFuncName(F, InLTO)))
434 return E;
435 // Also use getPGOFuncName() so that we can find records from older profiles
436 if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO)))
437 return E;
438 }
439 Sorted = false;
440 finalizeSymtab();
441 return Error::success();
442}
443
444Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
445 if (Error E = addFuncName(PGOFuncName))
446 return E;
447 MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
448 // In ThinLTO, local function may have been promoted to global and have
449 // suffix ".llvm." added to the function name. We need to add the
450 // stripped function name to the symbol table so that we can find a match
451 // from profile.
452 //
453 // We may have other suffixes similar as ".llvm." which are needed to
454 // be stripped before the matching, but ".__uniq." suffix which is used
455 // to differentiate internal linkage functions in different modules
456 // should be kept. Now this is the only suffix with the pattern ".xxx"
457 // which is kept before matching.
458 const std::string UniqSuffix = ".__uniq.";
459 auto pos = PGOFuncName.find(UniqSuffix);
460 // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
461 // search '.' from the beginning.
462 if (pos != std::string::npos)
463 pos += UniqSuffix.length();
464 else
465 pos = 0;
466 pos = PGOFuncName.find('.', pos);
467 if (pos != std::string::npos && pos != 0) {
468 StringRef OtherFuncName = PGOFuncName.substr(0, pos);
469 if (Error E = addFuncName(OtherFuncName))
470 return E;
471 MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
472 }
473 return Error::success();
474}
475
477 finalizeSymtab();
478 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
479 return A.first < Address;
480 });
481 // Raw function pointer collected by value profiler may be from
482 // external functions that are not instrumented. They won't have
483 // mapping data to be used by the deserializer. Force the value to
484 // be 0 in this case.
485 if (It != AddrToMD5Map.end() && It->first == Address)
486 return (uint64_t)It->second;
487 return 0;
488}
489
491 SmallVector<StringRef, 0> Sorted(NameTab.keys());
492 llvm::sort(Sorted);
493 for (StringRef S : Sorted)
494 OS << S << '\n';
495}
496
498 bool doCompression, std::string &Result) {
499 assert(!NameStrs.empty() && "No name data to emit");
500
501 uint8_t Header[20], *P = Header;
502 std::string UncompressedNameStrings =
503 join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
504
505 assert(StringRef(UncompressedNameStrings)
506 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
507 "PGO name is invalid (contains separator token)");
508
509 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
510 P += EncLen;
511
512 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
513 EncLen = encodeULEB128(CompressedLen, P);
514 P += EncLen;
515 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
516 unsigned HeaderLen = P - &Header[0];
517 Result.append(HeaderStr, HeaderLen);
518 Result += InputStr;
519 return Error::success();
520 };
521
522 if (!doCompression) {
523 return WriteStringToResult(0, UncompressedNameStrings);
524 }
525
526 SmallVector<uint8_t, 128> CompressedNameStrings;
527 compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings),
528 CompressedNameStrings,
530
531 return WriteStringToResult(CompressedNameStrings.size(),
532 toStringRef(CompressedNameStrings));
533}
534
536 auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
537 StringRef NameStr =
538 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
539 return NameStr;
540}
541
543 std::string &Result, bool doCompression) {
544 std::vector<std::string> NameStrs;
545 for (auto *NameVar : NameVars) {
546 NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
547 }
549 NameStrs, compression::zlib::isAvailable() && doCompression, Result);
550}
551
553 const uint8_t *P = NameStrings.bytes_begin();
554 const uint8_t *EndP = NameStrings.bytes_end();
555 while (P < EndP) {
556 uint32_t N;
557 uint64_t UncompressedSize = decodeULEB128(P, &N);
558 P += N;
559 uint64_t CompressedSize = decodeULEB128(P, &N);
560 P += N;
561 bool isCompressed = (CompressedSize != 0);
562 SmallVector<uint8_t, 128> UncompressedNameStrings;
563 StringRef NameStrings;
564 if (isCompressed) {
566 return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
567
568 if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
569 UncompressedNameStrings,
570 UncompressedSize)) {
571 consumeError(std::move(E));
572 return make_error<InstrProfError>(instrprof_error::uncompress_failed);
573 }
574 P += CompressedSize;
575 NameStrings = toStringRef(UncompressedNameStrings);
576 } else {
577 NameStrings =
578 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
579 P += UncompressedSize;
580 }
581 // Now parse the name strings.
583 NameStrings.split(Names, getInstrProfNameSeparator());
584 for (StringRef &Name : Names)
585 if (Error E = Symtab.addFuncName(Name))
586 return E;
587
588 while (P < EndP && *P == 0)
589 P++;
590 }
591 return Error::success();
592}
593
595 uint64_t FuncSum = 0;
596 Sum.NumEntries += Counts.size();
597 for (uint64_t Count : Counts)
598 FuncSum += Count;
599 Sum.CountSum += FuncSum;
600
601 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
602 uint64_t KindSum = 0;
603 uint32_t NumValueSites = getNumValueSites(VK);
604 for (size_t I = 0; I < NumValueSites; ++I) {
606 std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I);
607 for (uint32_t V = 0; V < NV; V++)
608 KindSum += VD[V].Count;
609 }
610 Sum.ValueCounts[VK] += KindSum;
611 }
612}
613
615 uint32_t ValueKind,
616 OverlapStats &Overlap,
617 OverlapStats &FuncLevelOverlap) {
618 this->sortByTargetValues();
619 Input.sortByTargetValues();
620 double Score = 0.0f, FuncLevelScore = 0.0f;
621 auto I = ValueData.begin();
622 auto IE = ValueData.end();
623 auto J = Input.ValueData.begin();
624 auto JE = Input.ValueData.end();
625 while (I != IE && J != JE) {
626 if (I->Value == J->Value) {
627 Score += OverlapStats::score(I->Count, J->Count,
628 Overlap.Base.ValueCounts[ValueKind],
629 Overlap.Test.ValueCounts[ValueKind]);
630 FuncLevelScore += OverlapStats::score(
631 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
632 FuncLevelOverlap.Test.ValueCounts[ValueKind]);
633 ++I;
634 } else if (I->Value < J->Value) {
635 ++I;
636 continue;
637 }
638 ++J;
639 }
640 Overlap.Overlap.ValueCounts[ValueKind] += Score;
641 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
642}
643
644// Return false on mismatch.
647 OverlapStats &Overlap,
648 OverlapStats &FuncLevelOverlap) {
649 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
650 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));
651 if (!ThisNumValueSites)
652 return;
653
654 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
655 getOrCreateValueSitesForKind(ValueKind);
657 Other.getValueSitesForKind(ValueKind);
658 for (uint32_t I = 0; I < ThisNumValueSites; I++)
659 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
660 FuncLevelOverlap);
661}
662
664 OverlapStats &FuncLevelOverlap,
665 uint64_t ValueCutoff) {
666 // FuncLevel CountSum for other should already computed and nonzero.
667 assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
668 accumulateCounts(FuncLevelOverlap.Base);
669 bool Mismatch = (Counts.size() != Other.Counts.size());
670
671 // Check if the value profiles mismatch.
672 if (!Mismatch) {
673 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
674 uint32_t ThisNumValueSites = getNumValueSites(Kind);
675 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
676 if (ThisNumValueSites != OtherNumValueSites) {
677 Mismatch = true;
678 break;
679 }
680 }
681 }
682 if (Mismatch) {
683 Overlap.addOneMismatch(FuncLevelOverlap.Test);
684 return;
685 }
686
687 // Compute overlap for value counts.
688 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
689 overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
690
691 double Score = 0.0;
692 uint64_t MaxCount = 0;
693 // Compute overlap for edge counts.
694 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
695 Score += OverlapStats::score(Counts[I], Other.Counts[I],
696 Overlap.Base.CountSum, Overlap.Test.CountSum);
697 MaxCount = std::max(Other.Counts[I], MaxCount);
698 }
699 Overlap.Overlap.CountSum += Score;
700 Overlap.Overlap.NumEntries += 1;
701
702 if (MaxCount >= ValueCutoff) {
703 double FuncScore = 0.0;
704 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
705 FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
706 FuncLevelOverlap.Base.CountSum,
707 FuncLevelOverlap.Test.CountSum);
708 FuncLevelOverlap.Overlap.CountSum = FuncScore;
709 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
710 FuncLevelOverlap.Valid = true;
711 }
712}
713
715 uint64_t Weight,
716 function_ref<void(instrprof_error)> Warn) {
717 this->sortByTargetValues();
718 Input.sortByTargetValues();
719 auto I = ValueData.begin();
720 auto IE = ValueData.end();
721 for (const InstrProfValueData &J : Input.ValueData) {
722 while (I != IE && I->Value < J.Value)
723 ++I;
724 if (I != IE && I->Value == J.Value) {
725 bool Overflowed;
726 I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed);
727 if (Overflowed)
729 ++I;
730 continue;
731 }
732 ValueData.insert(I, J);
733 }
734}
735
737 function_ref<void(instrprof_error)> Warn) {
738 for (InstrProfValueData &I : ValueData) {
739 bool Overflowed;
740 I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
741 if (Overflowed)
743 }
744}
745
746// Merge Value Profile data from Src record to this record for ValueKind.
747// Scale merged value counts by \p Weight.
748void InstrProfRecord::mergeValueProfData(
749 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
750 function_ref<void(instrprof_error)> Warn) {
751 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
752 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
753 if (ThisNumValueSites != OtherNumValueSites) {
755 return;
756 }
757 if (!ThisNumValueSites)
758 return;
759 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
760 getOrCreateValueSitesForKind(ValueKind);
762 Src.getValueSitesForKind(ValueKind);
763 for (uint32_t I = 0; I < ThisNumValueSites; I++)
764 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
765}
766
768 function_ref<void(instrprof_error)> Warn) {
769 // If the number of counters doesn't match we either have bad data
770 // or a hash collision.
771 if (Counts.size() != Other.Counts.size()) {
773 return;
774 }
775
776 // Special handling of the first count as the PseudoCount.
777 CountPseudoKind OtherKind = Other.getCountPseudoKind();
779 if (OtherKind != NotPseudo || ThisKind != NotPseudo) {
780 // We don't allow the merge of a profile with pseudo counts and
781 // a normal profile (i.e. without pesudo counts).
782 // Profile supplimenation should be done after the profile merge.
783 if (OtherKind == NotPseudo || ThisKind == NotPseudo) {
785 return;
786 }
787 if (OtherKind == PseudoHot || ThisKind == PseudoHot)
789 else
791 return;
792 }
793
794 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
795 bool Overflowed;
797 SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
800 Overflowed = true;
801 }
802 Counts[I] = Value;
803 if (Overflowed)
805 }
806
807 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
808 mergeValueProfData(Kind, Other, Weight, Warn);
809}
810
811void InstrProfRecord::scaleValueProfData(
812 uint32_t ValueKind, uint64_t N, uint64_t D,
813 function_ref<void(instrprof_error)> Warn) {
814 for (auto &R : getValueSitesForKind(ValueKind))
815 R.scale(N, D, Warn);
816}
817
819 function_ref<void(instrprof_error)> Warn) {
820 assert(D != 0 && "D cannot be 0");
821 for (auto &Count : this->Counts) {
822 bool Overflowed;
823 Count = SaturatingMultiply(Count, N, &Overflowed) / D;
824 if (Count > getInstrMaxCountValue()) {
825 Count = getInstrMaxCountValue();
826 Overflowed = true;
827 }
828 if (Overflowed)
830 }
831 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
832 scaleValueProfData(Kind, N, D, Warn);
833}
834
835// Map indirect call target name hash to name string.
836uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
837 InstrProfSymtab *SymTab) {
838 if (!SymTab)
839 return Value;
840
841 if (ValueKind == IPVK_IndirectCallTarget)
842 return SymTab->getFunctionHashFromAddress(Value);
843
844 return Value;
845}
846
848 InstrProfValueData *VData, uint32_t N,
850 for (uint32_t I = 0; I < N; I++) {
851 VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
852 }
853 std::vector<InstrProfValueSiteRecord> &ValueSites =
854 getOrCreateValueSitesForKind(ValueKind);
855 if (N == 0)
856 ValueSites.emplace_back();
857 else
858 ValueSites.emplace_back(VData, VData + N);
859}
860
863 using IDT = BPFunctionNode::IDT;
864 using UtilityNodeT = BPFunctionNode::UtilityNodeT;
865 // Collect all function IDs ordered by their smallest timestamp. This will be
866 // used as the initial FunctionNode order.
867 SetVector<IDT> FunctionIds;
868 size_t LargestTraceSize = 0;
869 for (auto &Trace : Traces)
870 LargestTraceSize =
871 std::max(LargestTraceSize, Trace.FunctionNameRefs.size());
872 for (size_t Timestamp = 0; Timestamp < LargestTraceSize; Timestamp++)
873 for (auto &Trace : Traces)
874 if (Timestamp < Trace.FunctionNameRefs.size())
875 FunctionIds.insert(Trace.FunctionNameRefs[Timestamp]);
876
877 int N = std::ceil(std::log2(LargestTraceSize));
878
879 // TODO: We need to use the Trace.Weight field to give more weight to more
880 // important utilities
882 for (size_t TraceIdx = 0; TraceIdx < Traces.size(); TraceIdx++) {
883 auto &Trace = Traces[TraceIdx].FunctionNameRefs;
884 for (size_t Timestamp = 0; Timestamp < Trace.size(); Timestamp++) {
885 for (int I = std::floor(std::log2(Timestamp + 1)); I < N; I++) {
886 auto &FunctionId = Trace[Timestamp];
887 UtilityNodeT GroupId = TraceIdx * N + I;
888 FuncGroups[FunctionId].push_back(GroupId);
889 }
890 }
891 }
892
893 std::vector<BPFunctionNode> Nodes;
894 for (auto &Id : FunctionIds) {
895 auto &UNs = FuncGroups[Id];
896 llvm::sort(UNs);
897 UNs.erase(std::unique(UNs.begin(), UNs.end()), UNs.end());
898 Nodes.emplace_back(Id, UNs);
899 }
900 return Nodes;
901}
902
903#define INSTR_PROF_COMMON_API_IMPL
905
906/*!
907 * ValueProfRecordClosure Interface implementation for InstrProfRecord
908 * class. These C wrappers are used as adaptors so that C++ code can be
909 * invoked as callbacks.
910 */
912 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
913}
914
916 return reinterpret_cast<const InstrProfRecord *>(Record)
917 ->getNumValueSites(VKind);
918}
919
921 return reinterpret_cast<const InstrProfRecord *>(Record)
922 ->getNumValueData(VKind);
923}
924
926 uint32_t S) {
927 return reinterpret_cast<const InstrProfRecord *>(R)
928 ->getNumValueDataForSite(VK, S);
929}
930
931void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
932 uint32_t K, uint32_t S) {
933 reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
934}
935
936ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
937 ValueProfData *VD =
938 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
939 memset(VD, 0, TotalSizeInBytes);
940 return VD;
941}
942
943static ValueProfRecordClosure InstrProfRecordClosure = {
944 nullptr,
949 nullptr,
952
953// Wrapper implementation using the closure mechanism.
954uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
955 auto Closure = InstrProfRecordClosure;
956 Closure.Record = &Record;
957 return getValueProfDataSize(&Closure);
958}
959
960// Wrapper implementation using the closure mechanism.
961std::unique_ptr<ValueProfData>
962ValueProfData::serializeFrom(const InstrProfRecord &Record) {
964
965 std::unique_ptr<ValueProfData> VPD(
966 serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
967 return VPD;
968}
969
970void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
971 InstrProfSymtab *SymTab) {
972 Record.reserveSites(Kind, NumValueSites);
973
974 InstrProfValueData *ValueData = getValueProfRecordValueData(this);
975 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
976 uint8_t ValueDataCount = this->SiteCountArray[VSite];
977 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
978 ValueData += ValueDataCount;
979 }
980}
981
982// For writing/serializing, Old is the host endianness, and New is
983// byte order intended on disk. For Reading/deserialization, Old
984// is the on-disk source endianness, and New is the host endianness.
985void ValueProfRecord::swapBytes(support::endianness Old,
987 using namespace support;
988
989 if (Old == New)
990 return;
991
992 if (getHostEndianness() != Old) {
993 sys::swapByteOrder<uint32_t>(NumValueSites);
994 sys::swapByteOrder<uint32_t>(Kind);
995 }
996 uint32_t ND = getValueProfRecordNumValueData(this);
997 InstrProfValueData *VD = getValueProfRecordValueData(this);
998
999 // No need to swap byte array: SiteCountArrray.
1000 for (uint32_t I = 0; I < ND; I++) {
1001 sys::swapByteOrder<uint64_t>(VD[I].Value);
1002 sys::swapByteOrder<uint64_t>(VD[I].Count);
1003 }
1004 if (getHostEndianness() == Old) {
1005 sys::swapByteOrder<uint32_t>(NumValueSites);
1006 sys::swapByteOrder<uint32_t>(Kind);
1007 }
1008}
1009
1010void ValueProfData::deserializeTo(InstrProfRecord &Record,
1011 InstrProfSymtab *SymTab) {
1012 if (NumValueKinds == 0)
1013 return;
1014
1015 ValueProfRecord *VR = getFirstValueProfRecord(this);
1016 for (uint32_t K = 0; K < NumValueKinds; K++) {
1017 VR->deserializeTo(Record, SymTab);
1018 VR = getValueProfRecordNext(VR);
1019 }
1020}
1021
1022template <class T>
1023static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
1024 using namespace support;
1025
1026 if (Orig == little)
1027 return endian::readNext<T, little, unaligned>(D);
1028 else
1029 return endian::readNext<T, big, unaligned>(D);
1030}
1031
1032static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
1033 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
1034 ValueProfData());
1035}
1036
1037Error ValueProfData::checkIntegrity() {
1038 if (NumValueKinds > IPVK_Last + 1)
1039 return make_error<InstrProfError>(
1040 instrprof_error::malformed, "number of value profile kinds is invalid");
1041 // Total size needs to be multiple of quadword size.
1042 if (TotalSize % sizeof(uint64_t))
1043 return make_error<InstrProfError>(
1044 instrprof_error::malformed, "total size is not multiples of quardword");
1045
1046 ValueProfRecord *VR = getFirstValueProfRecord(this);
1047 for (uint32_t K = 0; K < this->NumValueKinds; K++) {
1048 if (VR->Kind > IPVK_Last)
1049 return make_error<InstrProfError>(instrprof_error::malformed,
1050 "value kind is invalid");
1051 VR = getValueProfRecordNext(VR);
1052 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
1053 return make_error<InstrProfError>(
1055 "value profile address is greater than total size");
1056 }
1057 return Error::success();
1058}
1059
1061ValueProfData::getValueProfData(const unsigned char *D,
1062 const unsigned char *const BufferEnd,
1063 support::endianness Endianness) {
1064 using namespace support;
1065
1066 if (D + sizeof(ValueProfData) > BufferEnd)
1067 return make_error<InstrProfError>(instrprof_error::truncated);
1068
1069 const unsigned char *Header = D;
1070 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
1071 if (D + TotalSize > BufferEnd)
1072 return make_error<InstrProfError>(instrprof_error::too_large);
1073
1074 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
1075 memcpy(VPD.get(), D, TotalSize);
1076 // Byte swap.
1077 VPD->swapBytesToHost(Endianness);
1078
1079 Error E = VPD->checkIntegrity();
1080 if (E)
1081 return std::move(E);
1082
1083 return std::move(VPD);
1084}
1085
1086void ValueProfData::swapBytesToHost(support::endianness Endianness) {
1087 using namespace support;
1088
1089 if (Endianness == getHostEndianness())
1090 return;
1091
1092 sys::swapByteOrder<uint32_t>(TotalSize);
1093 sys::swapByteOrder<uint32_t>(NumValueKinds);
1094
1095 ValueProfRecord *VR = getFirstValueProfRecord(this);
1096 for (uint32_t K = 0; K < NumValueKinds; K++) {
1097 VR->swapBytes(Endianness, getHostEndianness());
1098 VR = getValueProfRecordNext(VR);
1099 }
1100}
1101
1102void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
1103 using namespace support;
1104
1105 if (Endianness == getHostEndianness())
1106 return;
1107
1108 ValueProfRecord *VR = getFirstValueProfRecord(this);
1109 for (uint32_t K = 0; K < NumValueKinds; K++) {
1110 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1111 VR->swapBytes(getHostEndianness(), Endianness);
1112 VR = NVR;
1113 }
1114 sys::swapByteOrder<uint32_t>(TotalSize);
1115 sys::swapByteOrder<uint32_t>(NumValueKinds);
1116}
1117
1119 const InstrProfRecord &InstrProfR,
1120 InstrProfValueKind ValueKind, uint32_t SiteIdx,
1121 uint32_t MaxMDCount) {
1122 uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
1123 if (!NV)
1124 return;
1125
1126 uint64_t Sum = 0;
1127 std::unique_ptr<InstrProfValueData[]> VD =
1128 InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
1129
1130 ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
1131 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1132}
1133
1136 uint64_t Sum, InstrProfValueKind ValueKind,
1137 uint32_t MaxMDCount) {
1138 LLVMContext &Ctx = M.getContext();
1139 MDBuilder MDHelper(Ctx);
1141 // Tag
1142 Vals.push_back(MDHelper.createString("VP"));
1143 // Value Kind
1144 Vals.push_back(MDHelper.createConstant(
1145 ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
1146 // Total Count
1147 Vals.push_back(
1148 MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
1149
1150 // Value Profile Data
1151 uint32_t MDCount = MaxMDCount;
1152 for (auto &VD : VDs) {
1153 Vals.push_back(MDHelper.createConstant(
1154 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
1155 Vals.push_back(MDHelper.createConstant(
1156 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
1157 if (--MDCount == 0)
1158 break;
1159 }
1160 Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
1161}
1162
1164 InstrProfValueKind ValueKind,
1165 uint32_t MaxNumValueData,
1166 InstrProfValueData ValueData[],
1167 uint32_t &ActualNumValueData, uint64_t &TotalC,
1168 bool GetNoICPValue) {
1169 MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1170 if (!MD)
1171 return false;
1172
1173 unsigned NOps = MD->getNumOperands();
1174
1175 if (NOps < 5)
1176 return false;
1177
1178 // Operand 0 is a string tag "VP":
1179 MDString *Tag = cast<MDString>(MD->getOperand(0));
1180 if (!Tag)
1181 return false;
1182
1183 if (!Tag->getString().equals("VP"))
1184 return false;
1185
1186 // Now check kind:
1187 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1188 if (!KindInt)
1189 return false;
1190 if (KindInt->getZExtValue() != ValueKind)
1191 return false;
1192
1193 // Get total count
1194 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1195 if (!TotalCInt)
1196 return false;
1197 TotalC = TotalCInt->getZExtValue();
1198
1199 ActualNumValueData = 0;
1200
1201 for (unsigned I = 3; I < NOps; I += 2) {
1202 if (ActualNumValueData >= MaxNumValueData)
1203 break;
1204 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1205 ConstantInt *Count =
1206 mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1207 if (!Value || !Count)
1208 return false;
1209 uint64_t CntValue = Count->getZExtValue();
1210 if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1211 continue;
1212 ValueData[ActualNumValueData].Value = Value->getZExtValue();
1213 ValueData[ActualNumValueData].Count = CntValue;
1214 ActualNumValueData++;
1215 }
1216 return true;
1217}
1218
1220 return F.getMetadata(getPGOFuncNameMetadataName());
1221}
1222
1224 // Only for internal linkage functions.
1225 if (PGOFuncName == F.getName())
1226 return;
1227 // Don't create duplicated meta-data.
1229 return;
1230 LLVMContext &C = F.getContext();
1231 MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1232 F.setMetadata(getPGOFuncNameMetadataName(), N);
1233}
1234
1235bool needsComdatForCounter(const Function &F, const Module &M) {
1236 if (F.hasComdat())
1237 return true;
1238
1239 if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1240 return false;
1241
1242 // See createPGOFuncNameVar for more details. To avoid link errors, profile
1243 // counters for function with available_externally linkage needs to be changed
1244 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1245 // created. Without using comdat, duplicate entries won't be removed by the
1246 // linker leading to increased data segement size and raw profile size. Even
1247 // worse, since the referenced counter from profile per-function data object
1248 // will be resolved to the common strong definition, the profile counts for
1249 // available_externally functions will end up being duplicated in raw profile
1250 // data. This can result in distorted profile as the counts of those dups
1251 // will be accumulated by the profile merger.
1252 GlobalValue::LinkageTypes Linkage = F.getLinkage();
1253 if (Linkage != GlobalValue::ExternalWeakLinkage &&
1255 return false;
1256
1257 return true;
1258}
1259
1260// Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1261bool isIRPGOFlagSet(const Module *M) {
1262 auto IRInstrVar =
1263 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1264 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1265 return false;
1266
1267 // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1268 // have the decl.
1269 if (IRInstrVar->isDeclaration())
1270 return true;
1271
1272 // Check if the flag is set.
1273 if (!IRInstrVar->hasInitializer())
1274 return false;
1275
1276 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1277 if (!InitVal)
1278 return false;
1279 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1280}
1281
1282// Check if we can safely rename this Comdat function.
1283bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1284 if (F.getName().empty())
1285 return false;
1286 if (!needsComdatForCounter(F, *(F.getParent())))
1287 return false;
1288 // Unsafe to rename the address-taken function (which can be used in
1289 // function comparison).
1290 if (CheckAddressTaken && F.hasAddressTaken())
1291 return false;
1292 // Only safe to do if this function may be discarded if it is not used
1293 // in the compilation unit.
1294 if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1295 return false;
1296
1297 // For AvailableExternallyLinkage functions.
1298 if (!F.hasComdat()) {
1300 return true;
1301 }
1302 return true;
1303}
1304
1305// Create the variable for the profile file name.
1306void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1307 if (InstrProfileOutput.empty())
1308 return;
1309 Constant *ProfileNameConst =
1310 ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1311 GlobalVariable *ProfileNameVar = new GlobalVariable(
1312 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1313 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1315 Triple TT(M.getTargetTriple());
1316 if (TT.supportsCOMDAT()) {
1318 ProfileNameVar->setComdat(M.getOrInsertComdat(
1319 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1320 }
1321}
1322
1323Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1324 const std::string &TestFilename,
1325 bool IsCS) {
1326 auto getProfileSum = [IsCS](const std::string &Filename,
1327 CountSumOrPercent &Sum) -> Error {
1328 // This function is only used from llvm-profdata that doesn't use any kind
1329 // of VFS. Just create a default RealFileSystem to read profiles.
1330 auto FS = vfs::getRealFileSystem();
1331 auto ReaderOrErr = InstrProfReader::create(Filename, *FS);
1332 if (Error E = ReaderOrErr.takeError()) {
1333 return E;
1334 }
1335 auto Reader = std::move(ReaderOrErr.get());
1336 Reader->accumulateCounts(Sum, IsCS);
1337 return Error::success();
1338 };
1339 auto Ret = getProfileSum(BaseFilename, Base);
1340 if (Ret)
1341 return Ret;
1342 Ret = getProfileSum(TestFilename, Test);
1343 if (Ret)
1344 return Ret;
1345 this->BaseFilename = &BaseFilename;
1346 this->TestFilename = &TestFilename;
1347 Valid = true;
1348 return Error::success();
1349}
1350
1352 Mismatch.NumEntries += 1;
1353 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1354 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1355 if (Test.ValueCounts[I] >= 1.0f)
1357 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1358 }
1359}
1360
1362 Unique.NumEntries += 1;
1363 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1364 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1365 if (Test.ValueCounts[I] >= 1.0f)
1366 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1367 }
1368}
1369
1371 if (!Valid)
1372 return;
1373
1374 const char *EntryName =
1375 (Level == ProgramLevel ? "functions" : "edge counters");
1376 if (Level == ProgramLevel) {
1377 OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1378 << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1379 } else {
1380 OS << "Function level:\n"
1381 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1382 }
1383
1384 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1385 if (Mismatch.NumEntries)
1386 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1387 << "\n";
1388 if (Unique.NumEntries)
1389 OS << " # of " << EntryName
1390 << " only in test_profile: " << Unique.NumEntries << "\n";
1391
1392 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1393 << "\n";
1394 if (Mismatch.NumEntries)
1395 OS << " Mismatched count percentage (Edge): "
1396 << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1397 if (Unique.NumEntries)
1398 OS << " Percentage of Edge profile only in test_profile: "
1399 << format("%.3f%%", Unique.CountSum * 100) << "\n";
1400 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1401 << "\n"
1402 << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1403 << "\n";
1404
1405 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1406 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1407 continue;
1408 char ProfileKindName[20];
1409 switch (I) {
1410 case IPVK_IndirectCallTarget:
1411 strncpy(ProfileKindName, "IndirectCall", 19);
1412 break;
1413 case IPVK_MemOPSize:
1414 strncpy(ProfileKindName, "MemOP", 19);
1415 break;
1416 default:
1417 snprintf(ProfileKindName, 19, "VP[%d]", I);
1418 break;
1419 }
1420 OS << " " << ProfileKindName
1421 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1422 << "\n";
1423 if (Mismatch.NumEntries)
1424 OS << " Mismatched count percentage (" << ProfileKindName
1425 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1426 if (Unique.NumEntries)
1427 OS << " Percentage of " << ProfileKindName
1428 << " profile only in test_profile: "
1429 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1430 OS << " " << ProfileKindName
1431 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1432 << "\n"
1433 << " " << ProfileKindName
1434 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1435 << "\n";
1436 }
1437}
1438
1439namespace IndexedInstrProf {
1440// A C++14 compatible version of the offsetof macro.
1441template <typename T1, typename T2>
1442inline size_t constexpr offsetOf(T1 T2::*Member) {
1443 constexpr T2 Object{};
1444 return size_t(&(Object.*Member)) - size_t(&Object);
1445}
1446
1447static inline uint64_t read(const unsigned char *Buffer, size_t Offset) {
1448 return *reinterpret_cast<const uint64_t *>(Buffer + Offset);
1449}
1450
1452 using namespace support;
1453 return endian::byte_swap<uint64_t, little>(Version);
1454}
1455
1456Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
1457 using namespace support;
1458 static_assert(std::is_standard_layout_v<Header>,
1459 "The header should be standard layout type since we use offset "
1460 "of fields to read.");
1461 Header H;
1462
1463 H.Magic = read(Buffer, offsetOf(&Header::Magic));
1464 // Check the magic number.
1465 uint64_t Magic = endian::byte_swap<uint64_t, little>(H.Magic);
1467 return make_error<InstrProfError>(instrprof_error::bad_magic);
1468
1469 // Read the version.
1470 H.Version = read(Buffer, offsetOf(&Header::Version));
1471 if (GET_VERSION(H.formatVersion()) >
1473 return make_error<InstrProfError>(instrprof_error::unsupported_version);
1474
1475 switch (GET_VERSION(H.formatVersion())) {
1476 // When a new field is added in the header add a case statement here to
1477 // populate it.
1478 static_assert(
1480 "Please update the reading code below if a new field has been added, "
1481 "if not add a case statement to fall through to the latest version.");
1482 case 10ull:
1483 H.TemporalProfTracesOffset =
1485 [[fallthrough]];
1486 case 9ull:
1487 H.BinaryIdOffset = read(Buffer, offsetOf(&Header::BinaryIdOffset));
1488 [[fallthrough]];
1489 case 8ull:
1490 H.MemProfOffset = read(Buffer, offsetOf(&Header::MemProfOffset));
1491 [[fallthrough]];
1492 default: // Version7 (when the backwards compatible header was introduced).
1493 H.HashType = read(Buffer, offsetOf(&Header::HashType));
1494 H.HashOffset = read(Buffer, offsetOf(&Header::HashOffset));
1495 }
1496
1497 return H;
1498}
1499
1500size_t Header::size() const {
1501 switch (GET_VERSION(formatVersion())) {
1502 // When a new field is added to the header add a case statement here to
1503 // compute the size as offset of the new field + size of the new field. This
1504 // relies on the field being added to the end of the list.
1506 "Please update the size computation below if a new field has "
1507 "been added to the header, if not add a case statement to "
1508 "fall through to the latest version.");
1509 case 10ull:
1512 case 9ull:
1514 case 8ull:
1516 default: // Version7 (when the backwards compatible header was introduced).
1518 }
1519}
1520
1521} // namespace IndexedInstrProf
1522
1523} // end namespace llvm
aarch64 promote const
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_UNUSED
Definition: Compiler.h:184
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::string Name
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Definition: InstrProf.cpp:80
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define H(x, y, z)
Definition: MD5.cpp:57
This file contains the declarations for metadata subclasses.
Module.h This file contains the declarations for the Module class.
#define P(N)
uint64_t Timestamp
Definition: Profile.cpp:320
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const char * name
Definition: SMEABIPass.cpp:49
raw_pwrite_stream & OS
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallString class.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2890
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:145
This is an important base class in LLVM.
Definition: Constant.h:41
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
void setComdat(Comdat *C)
Definition: Globals.cpp:196
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:404
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:591
bool isDiscardableIfUnused() const
Definition: GlobalValue.h:543
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:250
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:168
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
Definition: GlobalValue.h:50
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
@ ExternalWeakLinkage
ExternalWeak linkage description.
Definition: GlobalValue.h:57
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
static char ID
Definition: InstrProf.h:399
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:237
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:423
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:476
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Definition: InstrProf.cpp:490
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:486
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:302
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1521
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
Definition: MDBuilder.cpp:24
MDString * createString(StringRef Str)
Return the given string as metadata.
Definition: MDBuilder.cpp:20
Metadata node.
Definition: Metadata.h:950
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1303
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1416
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1309
A single uniqued string.
Definition: Metadata.h:611
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:499
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable's name.
Definition: Mangler.cpp:119
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
A vector that has set insertion semantics.
Definition: SetVector.h:57
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
size_t size() const
Definition: SmallVector.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
Definition: StringMap.h:213
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:704
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:222
const unsigned char * bytes_end() const
Definition: StringRef.h:118
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:575
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition: StringRef.h:613
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
bool startswith(StringRef Prefix) const
Definition: StringRef.h:261
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition: StringRef.h:301
const unsigned char * bytes_begin() const
Definition: StringRef.h:115
unsigned size() const
Definition: Trace.h:95
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:1002
ObjectFormatType
Definition: Triple.h:279
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
See the file comment.
Definition: ValueMap.h:84
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
static uint64_t read(const unsigned char *Buffer, size_t Offset)
Definition: InstrProf.cpp:1447
const uint64_t Magic
Definition: InstrProf.h:986
size_t constexpr offsetOf(T1 T2::*Member)
Definition: InstrProf.cpp:1442
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
Definition: Compression.h:39
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
Definition: Path.cpp:603
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:440
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1163
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:91
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
Definition: InstrProf.cpp:552
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Definition: InstrProf.cpp:342
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1223
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Definition: InstrProf.cpp:313
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:282
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
Definition: InstrProf.cpp:931
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:367
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
Definition: STLExtras.h:1984
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
Definition: LEB128.h:128
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1219
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
Definition: InstrProf.cpp:1032
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig)
Definition: InstrProf.cpp:1023
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:218
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition: InstrProf.h:65
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:423
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1118
std::pair< StringRef, StringRef > getParsedIRPGOFuncName(StringRef IRPGOFuncName)
Definition: InstrProf.cpp:360
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1652
InstrProfSectKind
Definition: InstrProf.h:59
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:535
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition: MathExtras.h:550
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:170
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
support::endianness getHostEndianness()
Definition: InstrProf.h:952
static std::optional< std::string > lookupPGOFuncName(const Function &F)
Definition: InstrProf.cpp:304
@ Other
Any other memory.
instrprof_error
Definition: InstrProf.h:320
InstrProfValueKind
Definition: InstrProf.h:254
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:504
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:185
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1919
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1235
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:915
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1283
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1306
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:497
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
Definition: InstrProf.cpp:925
static ValueProfRecordClosure InstrProfRecordClosure
Definition: InstrProf.cpp:943
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:378
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
Definition: InstrProf.cpp:253
static StringRef getStrippedSourceFileName(const Function &F)
Definition: InstrProf.cpp:268
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1261
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1041
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
Definition: Metadata.h:56
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
Definition: InstrProf.cpp:911
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
Definition: InstrProf.cpp:936
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:920
#define N
double ValueCounts[IPVK_Last - IPVK_First+1]
Definition: InstrProf.h:600
uint64_t formatVersion() const
Definition: InstrProf.cpp:1451
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1456
Profiling information for a single function.
Definition: InstrProf.h:691
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:645
std::vector< uint64_t > Counts
Definition: InstrProf.h:692
CountPseudoKind getCountPseudoKind() const
Definition: InstrProf.h:797
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:594
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:906
void setPseudoCount(CountPseudoKind Kind)
Definition: InstrProf.h:805
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:767
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:847
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:910
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:663
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
Definition: InstrProf.h:916
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:818
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:669
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:714
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:614
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:661
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:736
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1351
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:644
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1323
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1370
CountSumOrPercent Overlap
Definition: InstrProf.h:618
CountSumOrPercent Base
Definition: InstrProf.h:614
uint64_t FuncHash
Definition: InstrProf.h:625
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1361
const std::string * BaseFilename
Definition: InstrProf.h:622
const std::string * TestFilename
Definition: InstrProf.h:623
CountSumOrPercent Unique
Definition: InstrProf.h:620
CountSumOrPercent Mismatch
Definition: InstrProf.h:619
StringRef FuncName
Definition: InstrProf.h:624
CountSumOrPercent Test
Definition: InstrProf.h:616
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...
Definition: InstrProf.cpp:861