LLVM 19.0.0git
InstrProf.cpp
Go to the documentation of this file.
1//===- InstrProf.cpp - Instrumented profiling format support --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for clang's instrumentation based PGO and
10// coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SetVector.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Config/config.h"
21#include "llvm/IR/Constant.h"
22#include "llvm/IR/Constants.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/LLVMContext.h"
28#include "llvm/IR/MDBuilder.h"
29#include "llvm/IR/Metadata.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/Type.h"
37#include "llvm/Support/Debug.h"
38#include "llvm/Support/Endian.h"
39#include "llvm/Support/Error.h"
41#include "llvm/Support/LEB128.h"
43#include "llvm/Support/Path.h"
47#include <algorithm>
48#include <cassert>
49#include <cstddef>
50#include <cstdint>
51#include <cstring>
52#include <memory>
53#include <string>
54#include <system_error>
55#include <type_traits>
56#include <utility>
57#include <vector>
58
59using namespace llvm;
60
61#define DEBUG_TYPE "instrprof"
62
64 "static-func-full-module-prefix", cl::init(true), cl::Hidden,
65 cl::desc("Use full module build paths in the profile counter names for "
66 "static functions."));
67
68// This option is tailored to users that have different top-level directory in
69// profile-gen and profile-use compilation. Users need to specific the number
70// of levels to strip. A value larger than the number of directories in the
71// source file will strip all the directory names and only leave the basename.
72//
73// Note current ThinLTO module importing for the indirect-calls assumes
74// the source directory name not being stripped. A non-zero option value here
75// can potentially prevent some inter-module indirect-call-promotions.
77 "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
78 cl::desc("Strip specified level of directory name from source path in "
79 "the profile counter name for static functions."));
80
82 const std::string &ErrMsg = "") {
83 std::string Msg;
85
86 switch (Err) {
87 case instrprof_error::success:
88 OS << "success";
89 break;
90 case instrprof_error::eof:
91 OS << "end of File";
92 break;
93 case instrprof_error::unrecognized_format:
94 OS << "unrecognized instrumentation profile encoding format";
95 break;
96 case instrprof_error::bad_magic:
97 OS << "invalid instrumentation profile data (bad magic)";
98 break;
99 case instrprof_error::bad_header:
100 OS << "invalid instrumentation profile data (file header is corrupt)";
101 break;
102 case instrprof_error::unsupported_version:
103 OS << "unsupported instrumentation profile format version";
104 break;
105 case instrprof_error::unsupported_hash_type:
106 OS << "unsupported instrumentation profile hash type";
107 break;
108 case instrprof_error::too_large:
109 OS << "too much profile data";
110 break;
111 case instrprof_error::truncated:
112 OS << "truncated profile data";
113 break;
114 case instrprof_error::malformed:
115 OS << "malformed instrumentation profile data";
116 break;
117 case instrprof_error::missing_correlation_info:
118 OS << "debug info/binary for correlation is required";
119 break;
120 case instrprof_error::unexpected_correlation_info:
121 OS << "debug info/binary for correlation is not necessary";
122 break;
123 case instrprof_error::unable_to_correlate_profile:
124 OS << "unable to correlate profile";
125 break;
126 case instrprof_error::invalid_prof:
127 OS << "invalid profile created. Please file a bug "
128 "at: " BUG_REPORT_URL
129 " and include the profraw files that caused this error.";
130 break;
131 case instrprof_error::unknown_function:
132 OS << "no profile data available for function";
133 break;
134 case instrprof_error::hash_mismatch:
135 OS << "function control flow change detected (hash mismatch)";
136 break;
137 case instrprof_error::count_mismatch:
138 OS << "function basic block count change detected (counter mismatch)";
139 break;
140 case instrprof_error::bitmap_mismatch:
141 OS << "function bitmap size change detected (bitmap size mismatch)";
142 break;
143 case instrprof_error::counter_overflow:
144 OS << "counter overflow";
145 break;
146 case instrprof_error::value_site_count_mismatch:
147 OS << "function value site count change detected (counter mismatch)";
148 break;
149 case instrprof_error::compress_failed:
150 OS << "failed to compress data (zlib)";
151 break;
152 case instrprof_error::uncompress_failed:
153 OS << "failed to uncompress data (zlib)";
154 break;
155 case instrprof_error::empty_raw_profile:
156 OS << "empty raw profile file";
157 break;
158 case instrprof_error::zlib_unavailable:
159 OS << "profile uses zlib compression but the profile reader was built "
160 "without zlib support";
161 break;
162 case instrprof_error::raw_profile_version_mismatch:
163 OS << "raw profile version mismatch";
164 break;
165 case instrprof_error::counter_value_too_large:
166 OS << "excessively large counter value suggests corrupted profile data";
167 break;
168 }
169
170 // If optional error message is not empty, append it to the message.
171 if (!ErrMsg.empty())
172 OS << ": " << ErrMsg;
173
174 return OS.str();
175}
176
177namespace {
178
179// FIXME: This class is only here to support the transition to llvm::Error. It
180// will be removed once this transition is complete. Clients should prefer to
181// deal with the Error value directly, rather than converting to error_code.
182class InstrProfErrorCategoryType : public std::error_category {
183 const char *name() const noexcept override { return "llvm.instrprof"; }
184
185 std::string message(int IE) const override {
186 return getInstrProfErrString(static_cast<instrprof_error>(IE));
187 }
188};
189
190} // end anonymous namespace
191
192const std::error_category &llvm::instrprof_category() {
193 static InstrProfErrorCategoryType ErrorCategory;
194 return ErrorCategory;
195}
196
197namespace {
198
199const char *InstrProfSectNameCommon[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
201 SectNameCommon,
203};
204
205const char *InstrProfSectNameCoff[] = {
206#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
207 SectNameCoff,
209};
210
211const char *InstrProfSectNamePrefix[] = {
212#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
213 Prefix,
215};
216
217} // namespace
218
219namespace llvm {
220
222 "enable-name-compression",
223 cl::desc("Enable name/filename string compression"), cl::init(true));
224
226 "enable-vtable-value-profiling", cl::init(false),
227 cl::desc("If true, the virtual table address will be instrumented to know "
228 "the types of a C++ pointer. The information is used in indirect "
229 "call promotion to do selective vtable-based comparison."));
230
233 bool AddSegmentInfo) {
234 std::string SectName;
235
236 if (OF == Triple::MachO && AddSegmentInfo)
237 SectName = InstrProfSectNamePrefix[IPSK];
238
239 if (OF == Triple::COFF)
240 SectName += InstrProfSectNameCoff[IPSK];
241 else
242 SectName += InstrProfSectNameCommon[IPSK];
243
244 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
245 SectName += ",regular,live_support";
246
247 return SectName;
248}
249
250std::string InstrProfError::message() const {
251 return getInstrProfErrString(Err, Msg);
252}
253
254char InstrProfError::ID = 0;
255
257 StringRef FileName,
259 // Value names may be prefixed with a binary '1' to indicate
260 // that the backend should not modify the symbols due to any platform
261 // naming convention. Do not include that '1' in the PGO profile name.
262 if (Name[0] == '\1')
263 Name = Name.substr(1);
264
265 std::string NewName = std::string(Name);
267 // For local symbols, prepend the main file name to distinguish them.
268 // Do not include the full path in the file name since there's no guarantee
269 // that it will stay the same, e.g., if the files are checked out from
270 // version control in different locations.
271 if (FileName.empty())
272 NewName = NewName.insert(0, "<unknown>:");
273 else
274 NewName = NewName.insert(0, FileName.str() + ":");
275 }
276 return NewName;
277}
278
279// Strip NumPrefix level of directory name from PathNameStr. If the number of
280// directory separators is less than NumPrefix, strip all the directories and
281// leave base file name only.
282static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
283 uint32_t Count = NumPrefix;
284 uint32_t Pos = 0, LastPos = 0;
285 for (const auto &CI : PathNameStr) {
286 ++Pos;
288 LastPos = Pos;
289 --Count;
290 }
291 if (Count == 0)
292 break;
293 }
294 return PathNameStr.substr(LastPos);
295}
296
298 StringRef FileName(GO.getParent()->getSourceFileName());
299 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
300 if (StripLevel < StaticFuncStripDirNamePrefix)
301 StripLevel = StaticFuncStripDirNamePrefix;
302 if (StripLevel)
303 FileName = stripDirPrefix(FileName, StripLevel);
304 return FileName;
305}
306
307// The PGO name has the format [<filepath>;]<mangled-name> where <filepath>; is
308// provided if linkage is local and is used to discriminate possibly identical
309// mangled names. ";" is used because it is unlikely to be found in either
310// <filepath> or <mangled-name>.
311//
312// Older compilers used getPGOFuncName() which has the format
313// [<filepath>:]<mangled-name>. This caused trouble for Objective-C functions
314// which commonly have :'s in their names. We still need to compute this name to
315// lookup functions from profiles built by older compilers.
316static std::string
319 StringRef FileName) {
320 return GlobalValue::getGlobalIdentifier(GO.getName(), Linkage, FileName);
321}
322
323static std::optional<std::string> lookupPGONameFromMetadata(MDNode *MD) {
324 if (MD != nullptr) {
325 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
326 return S.str();
327 }
328 return {};
329}
330
331// Returns the PGO object name. This function has some special handling
332// when called in LTO optimization. The following only applies when calling in
333// LTO passes (when \c InLTO is true): LTO's internalization privatizes many
334// global linkage symbols. This happens after value profile annotation, but
335// those internal linkage functions should not have a source prefix.
336// Additionally, for ThinLTO mode, exported internal functions are promoted
337// and renamed. We need to ensure that the original internal PGO name is
338// used when computing the GUID that is compared against the profiled GUIDs.
339// To differentiate compiler generated internal symbols from original ones,
340// PGOFuncName meta data are created and attached to the original internal
341// symbols in the value profile annotation step
342// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
343// data, its original linkage must be non-internal.
344static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO,
345 MDNode *PGONameMetadata) {
346 if (!InLTO) {
347 auto FileName = getStrippedSourceFileName(GO);
348 return getIRPGONameForGlobalObject(GO, GO.getLinkage(), FileName);
349 }
350
351 // In LTO mode (when InLTO is true), first check if there is a meta data.
352 if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata))
353 return *IRPGOFuncName;
354
355 // If there is no meta data, the function must be a global before the value
356 // profile annotation pass. Its current linkage may be internal if it is
357 // internalized in LTO mode.
359}
360
361// Returns the IRPGO function name and does special handling when called
362// in LTO optimization. See the comments of `getIRPGOObjectName` for details.
363std::string getIRPGOFuncName(const Function &F, bool InLTO) {
365}
366
367// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
368// for front-end (Clang, etc) instrumentation.
369// The implementation is kept for profile matching from older profiles.
370// This is similar to `getIRPGOFuncName` except that this function calls
371// 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls
372// 'getIRPGONameForGlobalObject'. See the difference between two callees in the
373// comments of `getIRPGONameForGlobalObject`.
374std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
375 if (!InLTO) {
376 auto FileName = getStrippedSourceFileName(F);
377 return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
378 }
379
380 // In LTO mode (when InLTO is true), first check if there is a meta data.
381 if (auto PGOFuncName = lookupPGONameFromMetadata(getPGOFuncNameMetadata(F)))
382 return *PGOFuncName;
383
384 // If there is no meta data, the function must be a global before the value
385 // profile annotation pass. Its current linkage may be internal if it is
386 // internalized in LTO mode.
387 return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
388}
389
390std::string getPGOName(const GlobalVariable &V, bool InLTO) {
391 // PGONameMetadata should be set by compiler at profile use time
392 // and read by symtab creation to look up symbols corresponding to
393 // a MD5 hash.
394 return getIRPGOObjectName(V, InLTO, /*PGONameMetadata=*/nullptr);
395}
396
397// See getIRPGOObjectName() for a discription of the format.
398std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName) {
399 auto [FileName, MangledName] = IRPGOName.split(kGlobalIdentifierDelimiter);
400 if (MangledName.empty())
401 return std::make_pair(StringRef(), IRPGOName);
402 return std::make_pair(FileName, MangledName);
403}
404
406 if (FileName.empty())
407 return PGOFuncName;
408 // Drop the file name including ':' or ';'. See getIRPGONameForGlobalObject as
409 // well.
410 if (PGOFuncName.starts_with(FileName))
411 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
412 return PGOFuncName;
413}
414
415// \p FuncName is the string used as profile lookup key for the function. A
416// symbol is created to hold the name. Return the legalized symbol name.
417std::string getPGOFuncNameVarName(StringRef FuncName,
419 std::string VarName = std::string(getInstrProfNameVarPrefix());
420 VarName += FuncName;
421
422 if (!GlobalValue::isLocalLinkage(Linkage))
423 return VarName;
424
425 // Now fix up illegal chars in local VarName that may upset the assembler.
426 const char InvalidChars[] = "-:;<>/\"'";
427 size_t found = VarName.find_first_of(InvalidChars);
428 while (found != std::string::npos) {
429 VarName[found] = '_';
430 found = VarName.find_first_of(InvalidChars, found + 1);
431 }
432 return VarName;
433}
434
437 StringRef PGOFuncName) {
438 // We generally want to match the function's linkage, but available_externally
439 // and extern_weak both have the wrong semantics, and anything that doesn't
440 // need to link across compilation units doesn't need to be visible at all.
443 else if (Linkage == GlobalValue::AvailableExternallyLinkage)
445 else if (Linkage == GlobalValue::InternalLinkage ||
448
449 auto *Value =
450 ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
451 auto FuncNameVar =
452 new GlobalVariable(M, Value->getType(), true, Linkage, Value,
453 getPGOFuncNameVarName(PGOFuncName, Linkage));
454
455 // Hide the symbol so that we correctly get a copy for each executable.
456 if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
457 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
458
459 return FuncNameVar;
460}
461
463 return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
464}
465
467 for (Function &F : M) {
468 // Function may not have a name: like using asm("") to overwrite the name.
469 // Ignore in this case.
470 if (!F.hasName())
471 continue;
472 if (Error E = addFuncWithName(F, getIRPGOFuncName(F, InLTO)))
473 return E;
474 // Also use getPGOFuncName() so that we can find records from older profiles
475 if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO)))
476 return E;
477 }
478
480 for (GlobalVariable &G : M.globals()) {
481 if (!G.hasName() || !G.hasMetadata(LLVMContext::MD_type))
482 continue;
483 if (Error E = addVTableWithName(
484 G, getIRPGOObjectName(G, InLTO, /* PGONameMetadata */ nullptr)))
485 return E;
486 }
487
488 Sorted = false;
489 finalizeSymtab();
490 return Error::success();
491}
492
493Error InstrProfSymtab::addVTableWithName(GlobalVariable &VTable,
494 StringRef VTablePGOName) {
495 auto mapName = [&](StringRef Name) -> Error {
496 if (Error E = addSymbolName(Name))
497 return E;
498
499 bool Inserted = true;
500 std::tie(std::ignore, Inserted) =
501 MD5VTableMap.try_emplace(GlobalValue::getGUID(Name), &VTable);
502 if (!Inserted)
503 LLVM_DEBUG(dbgs() << "GUID conflict within one module");
504 return Error::success();
505 };
506 if (Error E = mapName(VTablePGOName))
507 return E;
508
509 StringRef CanonicalName = getCanonicalName(VTablePGOName);
510 if (CanonicalName != VTablePGOName)
511 return mapName(CanonicalName);
512
513 return Error::success();
514}
515
516/// \c NameStrings is a string composed of one of more possibly encoded
517/// sub-strings. The substrings are separated by 0 or more zero bytes. This
518/// method decodes the string and calls `NameCallback` for each substring.
519static Error
521 std::function<Error(StringRef)> NameCallback) {
522 const uint8_t *P = NameStrings.bytes_begin();
523 const uint8_t *EndP = NameStrings.bytes_end();
524 while (P < EndP) {
525 uint32_t N;
526 uint64_t UncompressedSize = decodeULEB128(P, &N);
527 P += N;
528 uint64_t CompressedSize = decodeULEB128(P, &N);
529 P += N;
530 bool isCompressed = (CompressedSize != 0);
531 SmallVector<uint8_t, 128> UncompressedNameStrings;
532 StringRef NameStrings;
533 if (isCompressed) {
535 return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
536
537 if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
538 UncompressedNameStrings,
539 UncompressedSize)) {
540 consumeError(std::move(E));
541 return make_error<InstrProfError>(instrprof_error::uncompress_failed);
542 }
543 P += CompressedSize;
544 NameStrings = toStringRef(UncompressedNameStrings);
545 } else {
546 NameStrings =
547 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
548 P += UncompressedSize;
549 }
550 // Now parse the name strings.
552 NameStrings.split(Names, getInstrProfNameSeparator());
553 for (StringRef &Name : Names)
554 if (Error E = NameCallback(Name))
555 return E;
556
557 while (P < EndP && *P == 0)
558 P++;
559 }
560 return Error::success();
561}
562
565 NameStrings,
566 std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
567}
568
570 StringRef VTableNameStrings) {
571 if (Error E = readAndDecodeStrings(FuncNameStrings,
573 this, std::placeholders::_1)))
574 return E;
575
577 VTableNameStrings,
578 std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
579}
580
582 StringRef CompressedVTableStrings) {
584 CompressedVTableStrings,
585 std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
586}
587
588StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) {
589 // In ThinLTO, local function may have been promoted to global and have
590 // suffix ".llvm." added to the function name. We need to add the
591 // stripped function name to the symbol table so that we can find a match
592 // from profile.
593 //
594 // ".__uniq." suffix is used to differentiate internal linkage functions in
595 // different modules and should be kept. This is the only suffix with the
596 // pattern ".xxx" which is kept before matching, other suffixes similar as
597 // ".llvm." will be stripped.
598 const std::string UniqSuffix = ".__uniq.";
599 size_t pos = PGOName.find(UniqSuffix);
600 if (pos != StringRef::npos)
601 pos += UniqSuffix.length();
602 else
603 pos = 0;
604
605 // Search '.' after ".__uniq." if ".__uniq." exists, otherwise search '.' from
606 // the beginning.
607 pos = PGOName.find('.', pos);
608 if (pos != StringRef::npos && pos != 0)
609 return PGOName.substr(0, pos);
610
611 return PGOName;
612}
613
614Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) {
615 auto mapName = [&](StringRef Name) -> Error {
616 if (Error E = addFuncName(Name))
617 return E;
618 MD5FuncMap.emplace_back(Function::getGUID(Name), &F);
619 return Error::success();
620 };
621 if (Error E = mapName(PGOFuncName))
622 return E;
623
624 StringRef CanonicalFuncName = getCanonicalName(PGOFuncName);
625 if (CanonicalFuncName != PGOFuncName)
626 return mapName(CanonicalFuncName);
627
628 return Error::success();
629}
630
632 // Given a runtime address, look up the hash value in the interval map, and
633 // fallback to value 0 if a hash value is not found.
634 return VTableAddrMap.lookup(Address, 0);
635}
636
638 finalizeSymtab();
639 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
640 return A.first < Address;
641 });
642 // Raw function pointer collected by value profiler may be from
643 // external functions that are not instrumented. They won't have
644 // mapping data to be used by the deserializer. Force the value to
645 // be 0 in this case.
646 if (It != AddrToMD5Map.end() && It->first == Address)
647 return (uint64_t)It->second;
648 return 0;
649}
650
652 SmallVector<StringRef, 0> Sorted(NameTab.keys());
653 llvm::sort(Sorted);
654 for (StringRef S : Sorted)
655 OS << S << '\n';
656}
657
659 bool doCompression, std::string &Result) {
660 assert(!NameStrs.empty() && "No name data to emit");
661
662 uint8_t Header[20], *P = Header;
663 std::string UncompressedNameStrings =
664 join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
665
666 assert(StringRef(UncompressedNameStrings)
667 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
668 "PGO name is invalid (contains separator token)");
669
670 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
671 P += EncLen;
672
673 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
674 EncLen = encodeULEB128(CompressedLen, P);
675 P += EncLen;
676 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
677 unsigned HeaderLen = P - &Header[0];
678 Result.append(HeaderStr, HeaderLen);
679 Result += InputStr;
680 return Error::success();
681 };
682
683 if (!doCompression) {
684 return WriteStringToResult(0, UncompressedNameStrings);
685 }
686
687 SmallVector<uint8_t, 128> CompressedNameStrings;
688 compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings),
689 CompressedNameStrings,
691
692 return WriteStringToResult(CompressedNameStrings.size(),
693 toStringRef(CompressedNameStrings));
694}
695
697 auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
698 StringRef NameStr =
699 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
700 return NameStr;
701}
702
704 std::string &Result, bool doCompression) {
705 std::vector<std::string> NameStrs;
706 for (auto *NameVar : NameVars) {
707 NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
708 }
710 NameStrs, compression::zlib::isAvailable() && doCompression, Result);
711}
712
714 std::string &Result, bool doCompression) {
715 std::vector<std::string> VTableNameStrs;
716 for (auto *VTable : VTables)
717 VTableNameStrs.push_back(getPGOName(*VTable));
719 VTableNameStrs, compression::zlib::isAvailable() && doCompression,
720 Result);
721}
722
724 uint64_t FuncSum = 0;
725 Sum.NumEntries += Counts.size();
726 for (uint64_t Count : Counts)
727 FuncSum += Count;
728 Sum.CountSum += FuncSum;
729
730 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
731 uint64_t KindSum = 0;
732 uint32_t NumValueSites = getNumValueSites(VK);
733 for (size_t I = 0; I < NumValueSites; ++I) {
734 for (const auto &V : getValueArrayForSite(VK, I))
735 KindSum += V.Count;
736 }
737 Sum.ValueCounts[VK] += KindSum;
738 }
739}
740
742 uint32_t ValueKind,
743 OverlapStats &Overlap,
744 OverlapStats &FuncLevelOverlap) {
745 this->sortByTargetValues();
746 Input.sortByTargetValues();
747 double Score = 0.0f, FuncLevelScore = 0.0f;
748 auto I = ValueData.begin();
749 auto IE = ValueData.end();
750 auto J = Input.ValueData.begin();
751 auto JE = Input.ValueData.end();
752 while (I != IE && J != JE) {
753 if (I->Value == J->Value) {
754 Score += OverlapStats::score(I->Count, J->Count,
755 Overlap.Base.ValueCounts[ValueKind],
756 Overlap.Test.ValueCounts[ValueKind]);
757 FuncLevelScore += OverlapStats::score(
758 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
759 FuncLevelOverlap.Test.ValueCounts[ValueKind]);
760 ++I;
761 } else if (I->Value < J->Value) {
762 ++I;
763 continue;
764 }
765 ++J;
766 }
767 Overlap.Overlap.ValueCounts[ValueKind] += Score;
768 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
769}
770
771// Return false on mismatch.
774 OverlapStats &Overlap,
775 OverlapStats &FuncLevelOverlap) {
776 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
777 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));
778 if (!ThisNumValueSites)
779 return;
780
781 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
782 getOrCreateValueSitesForKind(ValueKind);
784 Other.getValueSitesForKind(ValueKind);
785 for (uint32_t I = 0; I < ThisNumValueSites; I++)
786 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
787 FuncLevelOverlap);
788}
789
791 OverlapStats &FuncLevelOverlap,
792 uint64_t ValueCutoff) {
793 // FuncLevel CountSum for other should already computed and nonzero.
794 assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
795 accumulateCounts(FuncLevelOverlap.Base);
796 bool Mismatch = (Counts.size() != Other.Counts.size());
797
798 // Check if the value profiles mismatch.
799 if (!Mismatch) {
800 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
801 uint32_t ThisNumValueSites = getNumValueSites(Kind);
802 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
803 if (ThisNumValueSites != OtherNumValueSites) {
804 Mismatch = true;
805 break;
806 }
807 }
808 }
809 if (Mismatch) {
810 Overlap.addOneMismatch(FuncLevelOverlap.Test);
811 return;
812 }
813
814 // Compute overlap for value counts.
815 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
816 overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
817
818 double Score = 0.0;
819 uint64_t MaxCount = 0;
820 // Compute overlap for edge counts.
821 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
822 Score += OverlapStats::score(Counts[I], Other.Counts[I],
823 Overlap.Base.CountSum, Overlap.Test.CountSum);
824 MaxCount = std::max(Other.Counts[I], MaxCount);
825 }
826 Overlap.Overlap.CountSum += Score;
827 Overlap.Overlap.NumEntries += 1;
828
829 if (MaxCount >= ValueCutoff) {
830 double FuncScore = 0.0;
831 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
832 FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
833 FuncLevelOverlap.Base.CountSum,
834 FuncLevelOverlap.Test.CountSum);
835 FuncLevelOverlap.Overlap.CountSum = FuncScore;
836 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
837 FuncLevelOverlap.Valid = true;
838 }
839}
840
842 uint64_t Weight,
843 function_ref<void(instrprof_error)> Warn) {
844 this->sortByTargetValues();
845 Input.sortByTargetValues();
846 auto I = ValueData.begin();
847 auto IE = ValueData.end();
848 std::vector<InstrProfValueData> Merged;
849 Merged.reserve(std::max(ValueData.size(), Input.ValueData.size()));
850 for (const InstrProfValueData &J : Input.ValueData) {
851 while (I != IE && I->Value < J.Value) {
852 Merged.push_back(*I);
853 ++I;
854 }
855 if (I != IE && I->Value == J.Value) {
856 bool Overflowed;
857 I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed);
858 if (Overflowed)
860 Merged.push_back(*I);
861 ++I;
862 continue;
863 }
864 Merged.push_back(J);
865 }
866 Merged.insert(Merged.end(), I, IE);
867 ValueData = std::move(Merged);
868}
869
871 function_ref<void(instrprof_error)> Warn) {
872 for (InstrProfValueData &I : ValueData) {
873 bool Overflowed;
874 I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
875 if (Overflowed)
877 }
878}
879
880// Merge Value Profile data from Src record to this record for ValueKind.
881// Scale merged value counts by \p Weight.
882void InstrProfRecord::mergeValueProfData(
883 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
884 function_ref<void(instrprof_error)> Warn) {
885 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
886 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
887 if (ThisNumValueSites != OtherNumValueSites) {
889 return;
890 }
891 if (!ThisNumValueSites)
892 return;
893 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
894 getOrCreateValueSitesForKind(ValueKind);
896 Src.getValueSitesForKind(ValueKind);
897 for (uint32_t I = 0; I < ThisNumValueSites; I++)
898 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
899}
900
902 function_ref<void(instrprof_error)> Warn) {
903 // If the number of counters doesn't match we either have bad data
904 // or a hash collision.
905 if (Counts.size() != Other.Counts.size()) {
907 return;
908 }
909
910 // Special handling of the first count as the PseudoCount.
911 CountPseudoKind OtherKind = Other.getCountPseudoKind();
913 if (OtherKind != NotPseudo || ThisKind != NotPseudo) {
914 // We don't allow the merge of a profile with pseudo counts and
915 // a normal profile (i.e. without pesudo counts).
916 // Profile supplimenation should be done after the profile merge.
917 if (OtherKind == NotPseudo || ThisKind == NotPseudo) {
919 return;
920 }
921 if (OtherKind == PseudoHot || ThisKind == PseudoHot)
923 else
925 return;
926 }
927
928 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
929 bool Overflowed;
931 SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
934 Overflowed = true;
935 }
936 Counts[I] = Value;
937 if (Overflowed)
939 }
940
941 // If the number of bitmap bytes doesn't match we either have bad data
942 // or a hash collision.
943 if (BitmapBytes.size() != Other.BitmapBytes.size()) {
945 return;
946 }
947
948 // Bitmap bytes are merged by simply ORing them together.
949 for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) {
950 BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I];
951 }
952
953 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
954 mergeValueProfData(Kind, Other, Weight, Warn);
955}
956
957void InstrProfRecord::scaleValueProfData(
958 uint32_t ValueKind, uint64_t N, uint64_t D,
959 function_ref<void(instrprof_error)> Warn) {
960 for (auto &R : getValueSitesForKind(ValueKind))
961 R.scale(N, D, Warn);
962}
963
965 function_ref<void(instrprof_error)> Warn) {
966 assert(D != 0 && "D cannot be 0");
967 for (auto &Count : this->Counts) {
968 bool Overflowed;
969 Count = SaturatingMultiply(Count, N, &Overflowed) / D;
970 if (Count > getInstrMaxCountValue()) {
971 Count = getInstrMaxCountValue();
972 Overflowed = true;
973 }
974 if (Overflowed)
976 }
977 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
978 scaleValueProfData(Kind, N, D, Warn);
979}
980
981// Map indirect call target name hash to name string.
982uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
983 InstrProfSymtab *SymTab) {
984 if (!SymTab)
985 return Value;
986
987 if (ValueKind == IPVK_IndirectCallTarget)
988 return SymTab->getFunctionHashFromAddress(Value);
989
990 if (ValueKind == IPVK_VTableTarget)
991 return SymTab->getVTableHashFromAddress(Value);
992
993 return Value;
994}
995
997 InstrProfValueData *VData, uint32_t N,
999 for (uint32_t I = 0; I < N; I++) {
1000 VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
1001 }
1002 std::vector<InstrProfValueSiteRecord> &ValueSites =
1003 getOrCreateValueSitesForKind(ValueKind);
1004 assert(ValueSites.size() == Site);
1005 if (N == 0)
1006 ValueSites.emplace_back();
1007 else
1008 ValueSites.emplace_back(VData, VData + N);
1009}
1010
1012 ArrayRef<TemporalProfTraceTy> Traces, std::vector<BPFunctionNode> &Nodes,
1013 bool RemoveOutlierUNs) {
1014 using IDT = BPFunctionNode::IDT;
1015 using UtilityNodeT = BPFunctionNode::UtilityNodeT;
1016 UtilityNodeT MaxUN = 0;
1017 DenseMap<IDT, size_t> IdToFirstTimestamp;
1018 DenseMap<IDT, UtilityNodeT> IdToFirstUN;
1020 // TODO: We need to use the Trace.Weight field to give more weight to more
1021 // important utilities
1022 for (auto &Trace : Traces) {
1023 size_t CutoffTimestamp = 1;
1024 for (size_t Timestamp = 0; Timestamp < Trace.FunctionNameRefs.size();
1025 Timestamp++) {
1026 IDT Id = Trace.FunctionNameRefs[Timestamp];
1027 auto [It, WasInserted] = IdToFirstTimestamp.try_emplace(Id, Timestamp);
1028 if (!WasInserted)
1029 It->getSecond() = std::min<size_t>(It->getSecond(), Timestamp);
1030 if (Timestamp >= CutoffTimestamp) {
1031 ++MaxUN;
1032 CutoffTimestamp = 2 * Timestamp;
1033 }
1034 IdToFirstUN.try_emplace(Id, MaxUN);
1035 }
1036 for (auto &[Id, FirstUN] : IdToFirstUN)
1037 for (auto UN = FirstUN; UN <= MaxUN; ++UN)
1038 IdToUNs[Id].push_back(UN);
1039 ++MaxUN;
1040 IdToFirstUN.clear();
1041 }
1042
1043 if (RemoveOutlierUNs) {
1045 for (auto &[Id, UNs] : IdToUNs)
1046 for (auto &UN : UNs)
1047 ++UNFrequency[UN];
1048 // Filter out utility nodes that are too infrequent or too prevalent to make
1049 // BalancedPartitioning more effective.
1050 for (auto &[Id, UNs] : IdToUNs)
1051 llvm::erase_if(UNs, [&](auto &UN) {
1052 return UNFrequency[UN] <= 1 || 2 * UNFrequency[UN] > IdToUNs.size();
1053 });
1054 }
1055
1056 for (auto &[Id, UNs] : IdToUNs)
1057 Nodes.emplace_back(Id, UNs);
1058
1059 // Since BalancedPartitioning is sensitive to the initial order, we explicitly
1060 // order nodes by their earliest timestamp.
1061 llvm::sort(Nodes, [&](auto &L, auto &R) {
1062 return std::make_pair(IdToFirstTimestamp[L.Id], L.Id) <
1063 std::make_pair(IdToFirstTimestamp[R.Id], R.Id);
1064 });
1065}
1066
1067#define INSTR_PROF_COMMON_API_IMPL
1069
1070/*!
1071 * ValueProfRecordClosure Interface implementation for InstrProfRecord
1072 * class. These C wrappers are used as adaptors so that C++ code can be
1073 * invoked as callbacks.
1074 */
1076 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
1077}
1078
1080 return reinterpret_cast<const InstrProfRecord *>(Record)
1081 ->getNumValueSites(VKind);
1082}
1083
1085 return reinterpret_cast<const InstrProfRecord *>(Record)
1086 ->getNumValueData(VKind);
1087}
1088
1090 uint32_t S) {
1091 const auto *IPR = reinterpret_cast<const InstrProfRecord *>(R);
1092 return IPR->getValueArrayForSite(VK, S).size();
1093}
1094
1095void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
1096 uint32_t K, uint32_t S) {
1097 const auto *IPR = reinterpret_cast<const InstrProfRecord *>(R);
1098 llvm::copy(IPR->getValueArrayForSite(K, S), Dst);
1099}
1100
1101ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
1102 ValueProfData *VD =
1103 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
1104 memset(VD, 0, TotalSizeInBytes);
1105 return VD;
1106}
1107
1108static ValueProfRecordClosure InstrProfRecordClosure = {
1109 nullptr,
1114 nullptr,
1117
1118// Wrapper implementation using the closure mechanism.
1119uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
1120 auto Closure = InstrProfRecordClosure;
1121 Closure.Record = &Record;
1122 return getValueProfDataSize(&Closure);
1123}
1124
1125// Wrapper implementation using the closure mechanism.
1126std::unique_ptr<ValueProfData>
1127ValueProfData::serializeFrom(const InstrProfRecord &Record) {
1129
1130 std::unique_ptr<ValueProfData> VPD(
1131 serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
1132 return VPD;
1133}
1134
1135void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
1136 InstrProfSymtab *SymTab) {
1137 Record.reserveSites(Kind, NumValueSites);
1138
1139 InstrProfValueData *ValueData = getValueProfRecordValueData(this);
1140 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1141 uint8_t ValueDataCount = this->SiteCountArray[VSite];
1142 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
1143 ValueData += ValueDataCount;
1144 }
1145}
1146
1147// For writing/serializing, Old is the host endianness, and New is
1148// byte order intended on disk. For Reading/deserialization, Old
1149// is the on-disk source endianness, and New is the host endianness.
1150void ValueProfRecord::swapBytes(llvm::endianness Old, llvm::endianness New) {
1151 using namespace support;
1152
1153 if (Old == New)
1154 return;
1155
1156 if (llvm::endianness::native != Old) {
1157 sys::swapByteOrder<uint32_t>(NumValueSites);
1158 sys::swapByteOrder<uint32_t>(Kind);
1159 }
1160 uint32_t ND = getValueProfRecordNumValueData(this);
1161 InstrProfValueData *VD = getValueProfRecordValueData(this);
1162
1163 // No need to swap byte array: SiteCountArrray.
1164 for (uint32_t I = 0; I < ND; I++) {
1165 sys::swapByteOrder<uint64_t>(VD[I].Value);
1166 sys::swapByteOrder<uint64_t>(VD[I].Count);
1167 }
1168 if (llvm::endianness::native == Old) {
1169 sys::swapByteOrder<uint32_t>(NumValueSites);
1170 sys::swapByteOrder<uint32_t>(Kind);
1171 }
1172}
1173
1174void ValueProfData::deserializeTo(InstrProfRecord &Record,
1175 InstrProfSymtab *SymTab) {
1176 if (NumValueKinds == 0)
1177 return;
1178
1179 ValueProfRecord *VR = getFirstValueProfRecord(this);
1180 for (uint32_t K = 0; K < NumValueKinds; K++) {
1181 VR->deserializeTo(Record, SymTab);
1182 VR = getValueProfRecordNext(VR);
1183 }
1184}
1185
1186static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
1187 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
1188 ValueProfData());
1189}
1190
1191Error ValueProfData::checkIntegrity() {
1192 if (NumValueKinds > IPVK_Last + 1)
1193 return make_error<InstrProfError>(
1194 instrprof_error::malformed, "number of value profile kinds is invalid");
1195 // Total size needs to be multiple of quadword size.
1196 if (TotalSize % sizeof(uint64_t))
1197 return make_error<InstrProfError>(
1198 instrprof_error::malformed, "total size is not multiples of quardword");
1199
1200 ValueProfRecord *VR = getFirstValueProfRecord(this);
1201 for (uint32_t K = 0; K < this->NumValueKinds; K++) {
1202 if (VR->Kind > IPVK_Last)
1203 return make_error<InstrProfError>(instrprof_error::malformed,
1204 "value kind is invalid");
1205 VR = getValueProfRecordNext(VR);
1206 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
1207 return make_error<InstrProfError>(
1209 "value profile address is greater than total size");
1210 }
1211 return Error::success();
1212}
1213
1215ValueProfData::getValueProfData(const unsigned char *D,
1216 const unsigned char *const BufferEnd,
1217 llvm::endianness Endianness) {
1218 using namespace support;
1219
1220 if (D + sizeof(ValueProfData) > BufferEnd)
1221 return make_error<InstrProfError>(instrprof_error::truncated);
1222
1223 const unsigned char *Header = D;
1224 uint32_t TotalSize = endian::readNext<uint32_t>(Header, Endianness);
1225
1226 if (D + TotalSize > BufferEnd)
1227 return make_error<InstrProfError>(instrprof_error::too_large);
1228
1229 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
1230 memcpy(VPD.get(), D, TotalSize);
1231 // Byte swap.
1232 VPD->swapBytesToHost(Endianness);
1233
1234 Error E = VPD->checkIntegrity();
1235 if (E)
1236 return std::move(E);
1237
1238 return std::move(VPD);
1239}
1240
1241void ValueProfData::swapBytesToHost(llvm::endianness Endianness) {
1242 using namespace support;
1243
1244 if (Endianness == llvm::endianness::native)
1245 return;
1246
1247 sys::swapByteOrder<uint32_t>(TotalSize);
1248 sys::swapByteOrder<uint32_t>(NumValueKinds);
1249
1250 ValueProfRecord *VR = getFirstValueProfRecord(this);
1251 for (uint32_t K = 0; K < NumValueKinds; K++) {
1252 VR->swapBytes(Endianness, llvm::endianness::native);
1253 VR = getValueProfRecordNext(VR);
1254 }
1255}
1256
1257void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) {
1258 using namespace support;
1259
1260 if (Endianness == llvm::endianness::native)
1261 return;
1262
1263 ValueProfRecord *VR = getFirstValueProfRecord(this);
1264 for (uint32_t K = 0; K < NumValueKinds; K++) {
1265 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1266 VR->swapBytes(llvm::endianness::native, Endianness);
1267 VR = NVR;
1268 }
1269 sys::swapByteOrder<uint32_t>(TotalSize);
1270 sys::swapByteOrder<uint32_t>(NumValueKinds);
1271}
1272
1274 const InstrProfRecord &InstrProfR,
1275 InstrProfValueKind ValueKind, uint32_t SiteIdx,
1276 uint32_t MaxMDCount) {
1277 auto VDs = InstrProfR.getValueArrayForSite(ValueKind, SiteIdx);
1278 if (VDs.empty())
1279 return;
1280 uint64_t Sum = 0;
1281 for (const InstrProfValueData &V : VDs)
1282 Sum = SaturatingAdd(Sum, V.Count);
1283 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1284}
1285
1288 uint64_t Sum, InstrProfValueKind ValueKind,
1289 uint32_t MaxMDCount) {
1290 if (VDs.empty())
1291 return;
1292 LLVMContext &Ctx = M.getContext();
1293 MDBuilder MDHelper(Ctx);
1295 // Tag
1296 Vals.push_back(MDHelper.createString("VP"));
1297 // Value Kind
1298 Vals.push_back(MDHelper.createConstant(
1299 ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
1300 // Total Count
1301 Vals.push_back(
1302 MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
1303
1304 // Value Profile Data
1305 uint32_t MDCount = MaxMDCount;
1306 for (const auto &VD : VDs) {
1307 Vals.push_back(MDHelper.createConstant(
1308 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
1309 Vals.push_back(MDHelper.createConstant(
1310 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
1311 if (--MDCount == 0)
1312 break;
1313 }
1314 Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
1315}
1316
1318 InstrProfValueKind ValueKind) {
1319 MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1320 if (!MD)
1321 return nullptr;
1322
1323 if (MD->getNumOperands() < 5)
1324 return nullptr;
1325
1326 MDString *Tag = cast<MDString>(MD->getOperand(0));
1327 if (!Tag || Tag->getString() != "VP")
1328 return nullptr;
1329
1330 // Now check kind:
1331 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1332 if (!KindInt)
1333 return nullptr;
1334 if (KindInt->getZExtValue() != ValueKind)
1335 return nullptr;
1336
1337 return MD;
1338}
1339
1340static bool getValueProfDataFromInstImpl(const MDNode *const MD,
1341 const uint32_t MaxNumDataWant,
1342 InstrProfValueData ValueData[],
1343 uint32_t &ActualNumValueData,
1344 uint64_t &TotalC, bool GetNoICPValue) {
1345 const unsigned NOps = MD->getNumOperands();
1346 // Get total count
1347 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1348 if (!TotalCInt)
1349 return false;
1350 TotalC = TotalCInt->getZExtValue();
1351 ActualNumValueData = 0;
1352
1353 for (unsigned I = 3; I < NOps; I += 2) {
1354 if (ActualNumValueData >= MaxNumDataWant)
1355 break;
1356 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1357 ConstantInt *Count =
1358 mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1359 if (!Value || !Count)
1360 return false;
1361 uint64_t CntValue = Count->getZExtValue();
1362 if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1363 continue;
1364 ValueData[ActualNumValueData].Value = Value->getZExtValue();
1365 ValueData[ActualNumValueData].Count = CntValue;
1366 ActualNumValueData++;
1367 }
1368 return true;
1369}
1370
1371std::unique_ptr<InstrProfValueData[]>
1373 uint32_t MaxNumValueData, uint32_t &ActualNumValueData,
1374 uint64_t &TotalC, bool GetNoICPValue) {
1375 MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind);
1376 if (!MD)
1377 return nullptr;
1378 auto ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumValueData);
1379 if (!getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueDataArray.get(),
1380 ActualNumValueData, TotalC, GetNoICPValue))
1381 return nullptr;
1382 return ValueDataArray;
1383}
1384
1386 return F.getMetadata(getPGOFuncNameMetadataName());
1387}
1388
1390 // Only for internal linkage functions.
1391 if (PGOFuncName == F.getName())
1392 return;
1393 // Don't create duplicated meta-data.
1395 return;
1396 LLVMContext &C = F.getContext();
1397 MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1398 F.setMetadata(getPGOFuncNameMetadataName(), N);
1399}
1400
1401bool needsComdatForCounter(const GlobalObject &GO, const Module &M) {
1402 if (GO.hasComdat())
1403 return true;
1404
1405 if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1406 return false;
1407
1408 // See createPGOFuncNameVar for more details. To avoid link errors, profile
1409 // counters for function with available_externally linkage needs to be changed
1410 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1411 // created. Without using comdat, duplicate entries won't be removed by the
1412 // linker leading to increased data segement size and raw profile size. Even
1413 // worse, since the referenced counter from profile per-function data object
1414 // will be resolved to the common strong definition, the profile counts for
1415 // available_externally functions will end up being duplicated in raw profile
1416 // data. This can result in distorted profile as the counts of those dups
1417 // will be accumulated by the profile merger.
1419 if (Linkage != GlobalValue::ExternalWeakLinkage &&
1421 return false;
1422
1423 return true;
1424}
1425
1426// Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1427bool isIRPGOFlagSet(const Module *M) {
1428 auto IRInstrVar =
1429 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1430 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1431 return false;
1432
1433 // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1434 // have the decl.
1435 if (IRInstrVar->isDeclaration())
1436 return true;
1437
1438 // Check if the flag is set.
1439 if (!IRInstrVar->hasInitializer())
1440 return false;
1441
1442 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1443 if (!InitVal)
1444 return false;
1445 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1446}
1447
1448// Check if we can safely rename this Comdat function.
1449bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1450 if (F.getName().empty())
1451 return false;
1452 if (!needsComdatForCounter(F, *(F.getParent())))
1453 return false;
1454 // Unsafe to rename the address-taken function (which can be used in
1455 // function comparison).
1456 if (CheckAddressTaken && F.hasAddressTaken())
1457 return false;
1458 // Only safe to do if this function may be discarded if it is not used
1459 // in the compilation unit.
1460 if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1461 return false;
1462
1463 // For AvailableExternallyLinkage functions.
1464 if (!F.hasComdat()) {
1466 return true;
1467 }
1468 return true;
1469}
1470
1471// Create the variable for the profile file name.
1472void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1473 if (InstrProfileOutput.empty())
1474 return;
1475 Constant *ProfileNameConst =
1476 ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1477 GlobalVariable *ProfileNameVar = new GlobalVariable(
1478 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1479 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1481 Triple TT(M.getTargetTriple());
1482 if (TT.supportsCOMDAT()) {
1484 ProfileNameVar->setComdat(M.getOrInsertComdat(
1485 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1486 }
1487}
1488
1489Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1490 const std::string &TestFilename,
1491 bool IsCS) {
1492 auto getProfileSum = [IsCS](const std::string &Filename,
1493 CountSumOrPercent &Sum) -> Error {
1494 // This function is only used from llvm-profdata that doesn't use any kind
1495 // of VFS. Just create a default RealFileSystem to read profiles.
1496 auto FS = vfs::getRealFileSystem();
1497 auto ReaderOrErr = InstrProfReader::create(Filename, *FS);
1498 if (Error E = ReaderOrErr.takeError()) {
1499 return E;
1500 }
1501 auto Reader = std::move(ReaderOrErr.get());
1502 Reader->accumulateCounts(Sum, IsCS);
1503 return Error::success();
1504 };
1505 auto Ret = getProfileSum(BaseFilename, Base);
1506 if (Ret)
1507 return Ret;
1508 Ret = getProfileSum(TestFilename, Test);
1509 if (Ret)
1510 return Ret;
1511 this->BaseFilename = &BaseFilename;
1512 this->TestFilename = &TestFilename;
1513 Valid = true;
1514 return Error::success();
1515}
1516
1518 Mismatch.NumEntries += 1;
1519 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1520 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1521 if (Test.ValueCounts[I] >= 1.0f)
1523 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1524 }
1525}
1526
1528 Unique.NumEntries += 1;
1529 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1530 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1531 if (Test.ValueCounts[I] >= 1.0f)
1532 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1533 }
1534}
1535
1537 if (!Valid)
1538 return;
1539
1540 const char *EntryName =
1541 (Level == ProgramLevel ? "functions" : "edge counters");
1542 if (Level == ProgramLevel) {
1543 OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1544 << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1545 } else {
1546 OS << "Function level:\n"
1547 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1548 }
1549
1550 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1551 if (Mismatch.NumEntries)
1552 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1553 << "\n";
1554 if (Unique.NumEntries)
1555 OS << " # of " << EntryName
1556 << " only in test_profile: " << Unique.NumEntries << "\n";
1557
1558 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1559 << "\n";
1560 if (Mismatch.NumEntries)
1561 OS << " Mismatched count percentage (Edge): "
1562 << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1563 if (Unique.NumEntries)
1564 OS << " Percentage of Edge profile only in test_profile: "
1565 << format("%.3f%%", Unique.CountSum * 100) << "\n";
1566 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1567 << "\n"
1568 << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1569 << "\n";
1570
1571 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1572 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1573 continue;
1574 char ProfileKindName[20] = {0};
1575 switch (I) {
1576 case IPVK_IndirectCallTarget:
1577 strncpy(ProfileKindName, "IndirectCall", 19);
1578 break;
1579 case IPVK_MemOPSize:
1580 strncpy(ProfileKindName, "MemOP", 19);
1581 break;
1582 case IPVK_VTableTarget:
1583 strncpy(ProfileKindName, "VTable", 19);
1584 break;
1585 default:
1586 snprintf(ProfileKindName, 19, "VP[%d]", I);
1587 break;
1588 }
1589 OS << " " << ProfileKindName
1590 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1591 << "\n";
1592 if (Mismatch.NumEntries)
1593 OS << " Mismatched count percentage (" << ProfileKindName
1594 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1595 if (Unique.NumEntries)
1596 OS << " Percentage of " << ProfileKindName
1597 << " profile only in test_profile: "
1598 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1599 OS << " " << ProfileKindName
1600 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1601 << "\n"
1602 << " " << ProfileKindName
1603 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1604 << "\n";
1605 }
1606}
1607
1608namespace IndexedInstrProf {
1609Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
1610 using namespace support;
1611 static_assert(std::is_standard_layout_v<Header>,
1612 "Use standard layout for Header for simplicity");
1613 Header H;
1614
1615 H.Magic = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1616 // Check the magic number.
1617 if (H.Magic != IndexedInstrProf::Magic)
1618 return make_error<InstrProfError>(instrprof_error::bad_magic);
1619
1620 // Read the version.
1621 H.Version = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1622 if (H.getIndexedProfileVersion() >
1624 return make_error<InstrProfError>(instrprof_error::unsupported_version);
1625
1627 "Please update the reader as needed when a new field is added "
1628 "or when indexed profile version gets bumped.");
1629
1630 Buffer += sizeof(uint64_t); // Skip Header.Unused field.
1631 H.HashType = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1632 H.HashOffset = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1633 if (H.getIndexedProfileVersion() >= 8)
1634 H.MemProfOffset =
1635 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1636 if (H.getIndexedProfileVersion() >= 9)
1637 H.BinaryIdOffset =
1638 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1639 // Version 11 is handled by this condition.
1640 if (H.getIndexedProfileVersion() >= 10)
1641 H.TemporalProfTracesOffset =
1642 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1643 if (H.getIndexedProfileVersion() >= 12)
1644 H.VTableNamesOffset =
1645 endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1646 return H;
1647}
1648
1650 return GET_VERSION(Version);
1651}
1652
1653size_t Header::size() const {
1654 switch (getIndexedProfileVersion()) {
1655 // To retain backward compatibility, new fields must be appended to the end
1656 // of the header, and byte offset of existing fields shouldn't change when
1657 // indexed profile version gets incremented.
1658 static_assert(
1660 "Please update the size computation below if a new field has "
1661 "been added to the header; for a version bump without new "
1662 "fields, add a case statement to fall through to the latest version.");
1663 case 12ull:
1664 return 72;
1665 case 11ull:
1666 [[fallthrough]];
1667 case 10ull:
1668 return 64;
1669 case 9ull:
1670 return 56;
1671 case 8ull:
1672 return 48;
1673 default: // Version7 (when the backwards compatible header was introduced).
1674 return 40;
1675 }
1676}
1677
1678} // namespace IndexedInstrProf
1679
1680} // end namespace llvm
aarch64 promote const
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_ATTRIBUTE_UNUSED
Definition: Compiler.h:203
This file contains the declarations for the subclasses of Constant, which represent the different fla...
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Definition: InstrProf.cpp:81
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
#define H(x, y, z)
Definition: MD5.cpp:57
This file contains the declarations for metadata subclasses.
Module.h This file contains the declarations for the Module class.
#define P(N)
uint64_t Timestamp
Definition: Profile.cpp:320
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const char * name
Definition: SMEABIPass.cpp:49
raw_pwrite_stream & OS
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2900
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:155
This is an important base class in LLVM.
Definition: Constant.h:41
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Definition: DenseMap.h:235
unsigned size() const
Definition: DenseMap.h:99
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
Tagged union holding either a T or a Error.
Definition: Error.h:481
void setComdat(Comdat *C)
Definition: Globals.cpp:202
bool hasComdat() const
Definition: GlobalObject.h:128
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:408
LinkageTypes getLinkage() const
Definition: GlobalValue.h:545
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:536
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:594
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
bool isDiscardableIfUnused() const
Definition: GlobalValue.h:547
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:67
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:253
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:174
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:50
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:59
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:58
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
Definition: GlobalValue.h:53
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:51
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:55
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:52
@ ExternalWeakLinkage
ExternalWeak linkage description.
Definition: GlobalValue.h:60
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:54
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
static char ID
Definition: InstrProf.h:419
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:250
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition: InstrProf.h:443
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:637
Error addSymbolName(StringRef SymbolName)
Definition: InstrProf.h:563
uint64_t getVTableHashFromAddress(uint64_t Address)
Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
Definition: InstrProf.cpp:631
Error addVTableName(StringRef VTableName)
Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.
Definition: InstrProf.h:585
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Definition: InstrProf.cpp:651
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
The method name is kept since there are many callers.
Definition: InstrProf.h:581
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)
Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.
Definition: InstrProf.cpp:581
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:375
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1635
ValT lookup(KeyT x, ValT NotFound=ValT()) const
lookup - Return the mapped value at x or NotFound.
Definition: IntervalMap.h:1119
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
Definition: MDBuilder.cpp:24
MDString * createString(StringRef Str)
Return the given string as metadata.
Definition: MDBuilder.cpp:20
Metadata node.
Definition: Metadata.h:1067
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1428
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1434
A single uniqued string.
Definition: Metadata.h:720
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:600
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const std::string & getSourceFileName() const
Get the module's original source file name.
Definition: Module.h:278
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
size_t size() const
Definition: SmallVector.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
Definition: StringMap.h:228
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:693
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:223
const unsigned char * bytes_end() const
Definition: StringRef.h:118
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:564
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:258
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition: StringRef.h:602
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition: StringRef.h:290
static constexpr size_t npos
Definition: StringRef.h:52
const unsigned char * bytes_begin() const
Definition: StringRef.h:115
unsigned size() const
Definition: Trace.h:95
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:1048
ObjectFormatType
Definition: Triple.h:297
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
See the file comment.
Definition: ValueMap.h:84
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:471
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
const uint64_t Magic
Definition: InstrProf.h:1079
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
Definition: Compression.h:39
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
Definition: Path.cpp:602
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:92
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
Definition: InstrProf.cpp:374
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1389
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Definition: InstrProf.cpp:363
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:297
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
Definition: InstrProf.cpp:1095
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:405
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
Definition: STLExtras.h:2008
std::unique_ptr< InstrProfValueData[]> getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
Definition: InstrProf.cpp:1372
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
Definition: LEB128.h:131
std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
Definition: InstrProf.cpp:398
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1385
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
Definition: InstrProf.cpp:1186
MDNode * mayHaveValueProfileOfKind(const Instruction &Inst, InstrProfValueKind ValueKind)
Definition: InstrProf.cpp:1317
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:231
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition: InstrProf.h:66
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1401
constexpr char kGlobalIdentifierDelimiter
Definition: GlobalValue.h:45
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
Definition: InstrProf.cpp:390
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:462
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1273
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
Definition: InstrProf.cpp:703
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
InstrProfSectKind
Definition: InstrProf.h:60
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:696
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition: MathExtras.h:610
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:178
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata)
Definition: InstrProf.cpp:344
@ Other
Any other memory.
instrprof_error
Definition: InstrProf.h:337
InstrProfValueKind
Definition: InstrProf.h:267
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:564
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:192
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
Definition: InstrProf.cpp:713
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1914
static StringRef getStrippedSourceFileName(const GlobalObject &GO)
Definition: InstrProf.cpp:297
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:1079
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1824
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1449
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1472
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
Definition: InstrProf.cpp:658
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2051
static bool getValueProfDataFromInstImpl(const MDNode *const MD, const uint32_t MaxNumDataWant, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue)
Definition: InstrProf.cpp:1340
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
Definition: InstrProf.cpp:1089
static ValueProfRecordClosure InstrProfRecordClosure
Definition: InstrProf.cpp:1108
static Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)
NameStrings is a string composed of one of more possibly encoded sub-strings.
Definition: InstrProf.cpp:520
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:417
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
Definition: InstrProf.cpp:282
endianness
Definition: bit.h:70
static std::optional< std::string > lookupPGONameFromMetadata(MDNode *MD)
Definition: InstrProf.cpp:323
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:535
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1427
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1069
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
Definition: Metadata.h:57
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
Definition: InstrProf.cpp:1075
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
Definition: InstrProf.cpp:1101
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:1084
static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName)
Definition: InstrProf.cpp:317
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
#define N
std::array< double, IPVK_Last - IPVK_First+1 > ValueCounts
Definition: InstrProf.h:729
uint64_t getIndexedProfileVersion() const
Definition: InstrProf.cpp:1649
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1609
Profiling information for a single function.
Definition: InstrProf.h:817
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:772
std::vector< uint64_t > Counts
Definition: InstrProf.h:818
ArrayRef< InstrProfValueData > getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const
Return the array of profiled values at Site.
Definition: InstrProf.h:1039
CountPseudoKind getCountPseudoKind() const
Definition: InstrProf.h:915
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:723
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:1034
void setPseudoCount(CountPseudoKind Kind)
Definition: InstrProf.h:923
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:901
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:996
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:790
std::vector< uint8_t > BitmapBytes
Definition: InstrProf.h:819
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:964
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:795
std::vector< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:787
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:841
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:741
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:870
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1517
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:770
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1489
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1536
CountSumOrPercent Overlap
Definition: InstrProf.h:746
CountSumOrPercent Base
Definition: InstrProf.h:742
uint64_t FuncHash
Definition: InstrProf.h:753
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1527
const std::string * BaseFilename
Definition: InstrProf.h:750
const std::string * TestFilename
Definition: InstrProf.h:751
CountSumOrPercent Unique
Definition: InstrProf.h:748
CountSumOrPercent Mismatch
Definition: InstrProf.h:747
StringRef FuncName
Definition: InstrProf.h:752
CountSumOrPercent Test
Definition: InstrProf.h:744
static void createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces, std::vector< BPFunctionNode > &Nodes, bool RemoveOutlierUNs=true)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...
Definition: InstrProf.cpp:1011