LLVM 17.0.0git
InstrProf.cpp
Go to the documentation of this file.
1//===- InstrProf.cpp - Instrumented profiling format support --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for clang's instrumentation based PGO and
10// coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/Config/config.h"
21#include "llvm/IR/Constant.h"
22#include "llvm/IR/Constants.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/LLVMContext.h"
28#include "llvm/IR/MDBuilder.h"
29#include "llvm/IR/Metadata.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/Type.h"
37#include "llvm/Support/Endian.h"
38#include "llvm/Support/Error.h"
40#include "llvm/Support/LEB128.h"
42#include "llvm/Support/Path.h"
46#include <algorithm>
47#include <cassert>
48#include <cstddef>
49#include <cstdint>
50#include <cstring>
51#include <memory>
52#include <string>
53#include <system_error>
54#include <type_traits>
55#include <utility>
56#include <vector>
57
58using namespace llvm;
59
61 "static-func-full-module-prefix", cl::init(true), cl::Hidden,
62 cl::desc("Use full module build paths in the profile counter names for "
63 "static functions."));
64
65// This option is tailored to users that have different top-level directory in
66// profile-gen and profile-use compilation. Users need to specific the number
67// of levels to strip. A value larger than the number of directories in the
68// source file will strip all the directory names and only leave the basename.
69//
70// Note current ThinLTO module importing for the indirect-calls assumes
71// the source directory name not being stripped. A non-zero option value here
72// can potentially prevent some inter-module indirect-call-promotions.
74 "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
75 cl::desc("Strip specified level of directory name from source path in "
76 "the profile counter name for static functions."));
77
79 const std::string &ErrMsg = "") {
80 std::string Msg;
82
83 switch (Err) {
84 case instrprof_error::success:
85 OS << "success";
86 break;
87 case instrprof_error::eof:
88 OS << "end of File";
89 break;
90 case instrprof_error::unrecognized_format:
91 OS << "unrecognized instrumentation profile encoding format";
92 break;
93 case instrprof_error::bad_magic:
94 OS << "invalid instrumentation profile data (bad magic)";
95 break;
96 case instrprof_error::bad_header:
97 OS << "invalid instrumentation profile data (file header is corrupt)";
98 break;
99 case instrprof_error::unsupported_version:
100 OS << "unsupported instrumentation profile format version";
101 break;
102 case instrprof_error::unsupported_hash_type:
103 OS << "unsupported instrumentation profile hash type";
104 break;
105 case instrprof_error::too_large:
106 OS << "too much profile data";
107 break;
108 case instrprof_error::truncated:
109 OS << "truncated profile data";
110 break;
111 case instrprof_error::malformed:
112 OS << "malformed instrumentation profile data";
113 break;
114 case instrprof_error::missing_debug_info_for_correlation:
115 OS << "debug info for correlation is required";
116 break;
117 case instrprof_error::unexpected_debug_info_for_correlation:
118 OS << "debug info for correlation is not necessary";
119 break;
120 case instrprof_error::unable_to_correlate_profile:
121 OS << "unable to correlate profile";
122 break;
123 case instrprof_error::invalid_prof:
124 OS << "invalid profile created. Please file a bug "
125 "at: " BUG_REPORT_URL
126 " and include the profraw files that caused this error.";
127 break;
128 case instrprof_error::unknown_function:
129 OS << "no profile data available for function";
130 break;
131 case instrprof_error::hash_mismatch:
132 OS << "function control flow change detected (hash mismatch)";
133 break;
134 case instrprof_error::count_mismatch:
135 OS << "function basic block count change detected (counter mismatch)";
136 break;
137 case instrprof_error::counter_overflow:
138 OS << "counter overflow";
139 break;
140 case instrprof_error::value_site_count_mismatch:
141 OS << "function value site count change detected (counter mismatch)";
142 break;
143 case instrprof_error::compress_failed:
144 OS << "failed to compress data (zlib)";
145 break;
146 case instrprof_error::uncompress_failed:
147 OS << "failed to uncompress data (zlib)";
148 break;
149 case instrprof_error::empty_raw_profile:
150 OS << "empty raw profile file";
151 break;
152 case instrprof_error::zlib_unavailable:
153 OS << "profile uses zlib compression but the profile reader was built "
154 "without zlib support";
155 break;
156 }
157
158 // If optional error message is not empty, append it to the message.
159 if (!ErrMsg.empty())
160 OS << ": " << ErrMsg;
161
162 return OS.str();
163}
164
165namespace {
166
167// FIXME: This class is only here to support the transition to llvm::Error. It
168// will be removed once this transition is complete. Clients should prefer to
169// deal with the Error value directly, rather than converting to error_code.
170class InstrProfErrorCategoryType : public std::error_category {
171 const char *name() const noexcept override { return "llvm.instrprof"; }
172
173 std::string message(int IE) const override {
174 return getInstrProfErrString(static_cast<instrprof_error>(IE));
175 }
176};
177
178} // end anonymous namespace
179
180const std::error_category &llvm::instrprof_category() {
181 static InstrProfErrorCategoryType ErrorCategory;
182 return ErrorCategory;
183}
184
185namespace {
186
187const char *InstrProfSectNameCommon[] = {
188#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
189 SectNameCommon,
191};
192
193const char *InstrProfSectNameCoff[] = {
194#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
195 SectNameCoff,
197};
198
199const char *InstrProfSectNamePrefix[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
201 Prefix,
203};
204
205} // namespace
206
207namespace llvm {
208
210 "enable-name-compression",
211 cl::desc("Enable name/filename string compression"), cl::init(true));
212
215 bool AddSegmentInfo) {
216 std::string SectName;
217
218 if (OF == Triple::MachO && AddSegmentInfo)
219 SectName = InstrProfSectNamePrefix[IPSK];
220
221 if (OF == Triple::COFF)
222 SectName += InstrProfSectNameCoff[IPSK];
223 else
224 SectName += InstrProfSectNameCommon[IPSK];
225
226 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
227 SectName += ",regular,live_support";
228
229 return SectName;
230}
231
233 if (IE == instrprof_error::success)
234 return;
235
236 if (FirstError == instrprof_error::success)
237 FirstError = IE;
238
239 switch (IE) {
241 ++NumHashMismatches;
242 break;
244 ++NumCountMismatches;
245 break;
247 ++NumCounterOverflows;
248 break;
250 ++NumValueSiteCountMismatches;
251 break;
252 default:
253 llvm_unreachable("Not a soft error");
254 }
255}
256
257std::string InstrProfError::message() const {
258 return getInstrProfErrString(Err, Msg);
259}
260
261char InstrProfError::ID = 0;
262
263std::string getPGOFuncName(StringRef RawFuncName,
265 StringRef FileName,
267 return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
268}
269
270// Strip NumPrefix level of directory name from PathNameStr. If the number of
271// directory separators is less than NumPrefix, strip all the directories and
272// leave base file name only.
273static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
274 uint32_t Count = NumPrefix;
275 uint32_t Pos = 0, LastPos = 0;
276 for (auto & CI : PathNameStr) {
277 ++Pos;
279 LastPos = Pos;
280 --Count;
281 }
282 if (Count == 0)
283 break;
284 }
285 return PathNameStr.substr(LastPos);
286}
287
288// Return the PGOFuncName. This function has some special handling when called
289// in LTO optimization. The following only applies when calling in LTO passes
290// (when \c InLTO is true): LTO's internalization privatizes many global linkage
291// symbols. This happens after value profile annotation, but those internal
292// linkage functions should not have a source prefix.
293// Additionally, for ThinLTO mode, exported internal functions are promoted
294// and renamed. We need to ensure that the original internal PGO name is
295// used when computing the GUID that is compared against the profiled GUIDs.
296// To differentiate compiler generated internal symbols from original ones,
297// PGOFuncName meta data are created and attached to the original internal
298// symbols in the value profile annotation step
299// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
300// data, its original linkage must be non-internal.
301std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
302 if (!InLTO) {
303 StringRef FileName(F.getParent()->getSourceFileName());
304 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
305 if (StripLevel < StaticFuncStripDirNamePrefix)
306 StripLevel = StaticFuncStripDirNamePrefix;
307 if (StripLevel)
308 FileName = stripDirPrefix(FileName, StripLevel);
309 return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
310 }
311
312 // In LTO mode (when InLTO is true), first check if there is a meta data.
313 if (MDNode *MD = getPGOFuncNameMetadata(F)) {
314 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
315 return S.str();
316 }
317
318 // If there is no meta data, the function must be a global before the value
319 // profile annotation pass. Its current linkage may be internal if it is
320 // internalized in LTO mode.
321 return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
322}
323
325 if (FileName.empty())
326 return PGOFuncName;
327 // Drop the file name including ':'. See also getPGOFuncName.
328 if (PGOFuncName.startswith(FileName))
329 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
330 return PGOFuncName;
331}
332
333// \p FuncName is the string used as profile lookup key for the function. A
334// symbol is created to hold the name. Return the legalized symbol name.
335std::string getPGOFuncNameVarName(StringRef FuncName,
337 std::string VarName = std::string(getInstrProfNameVarPrefix());
338 VarName += FuncName;
339
340 if (!GlobalValue::isLocalLinkage(Linkage))
341 return VarName;
342
343 // Now fix up illegal chars in local VarName that may upset the assembler.
344 const char *InvalidChars = "-:<>/\"'";
345 size_t found = VarName.find_first_of(InvalidChars);
346 while (found != std::string::npos) {
347 VarName[found] = '_';
348 found = VarName.find_first_of(InvalidChars, found + 1);
349 }
350 return VarName;
351}
352
355 StringRef PGOFuncName) {
356 // We generally want to match the function's linkage, but available_externally
357 // and extern_weak both have the wrong semantics, and anything that doesn't
358 // need to link across compilation units doesn't need to be visible at all.
361 else if (Linkage == GlobalValue::AvailableExternallyLinkage)
363 else if (Linkage == GlobalValue::InternalLinkage ||
366
367 auto *Value =
368 ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
369 auto FuncNameVar =
370 new GlobalVariable(M, Value->getType(), true, Linkage, Value,
371 getPGOFuncNameVarName(PGOFuncName, Linkage));
372
373 // Hide the symbol so that we correctly get a copy for each executable.
374 if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
375 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
376
377 return FuncNameVar;
378}
379
381 return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
382}
383
385 for (Function &F : M) {
386 // Function may not have a name: like using asm("") to overwrite the name.
387 // Ignore in this case.
388 if (!F.hasName())
389 continue;
390 const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
391 if (Error E = addFuncName(PGOFuncName))
392 return E;
393 MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
394 // In ThinLTO, local function may have been promoted to global and have
395 // suffix ".llvm." added to the function name. We need to add the
396 // stripped function name to the symbol table so that we can find a match
397 // from profile.
398 //
399 // We may have other suffixes similar as ".llvm." which are needed to
400 // be stripped before the matching, but ".__uniq." suffix which is used
401 // to differentiate internal linkage functions in different modules
402 // should be kept. Now this is the only suffix with the pattern ".xxx"
403 // which is kept before matching.
404 const std::string UniqSuffix = ".__uniq.";
405 auto pos = PGOFuncName.find(UniqSuffix);
406 // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
407 // search '.' from the beginning.
408 if (pos != std::string::npos)
409 pos += UniqSuffix.length();
410 else
411 pos = 0;
412 pos = PGOFuncName.find('.', pos);
413 if (pos != std::string::npos && pos != 0) {
414 const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
415 if (Error E = addFuncName(OtherFuncName))
416 return E;
417 MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
418 }
419 }
420 Sorted = false;
421 finalizeSymtab();
422 return Error::success();
423}
424
426 finalizeSymtab();
427 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
428 return A.first < Address;
429 });
430 // Raw function pointer collected by value profiler may be from
431 // external functions that are not instrumented. They won't have
432 // mapping data to be used by the deserializer. Force the value to
433 // be 0 in this case.
434 if (It != AddrToMD5Map.end() && It->first == Address)
435 return (uint64_t)It->second;
436 return 0;
437}
438
440 bool doCompression, std::string &Result) {
441 assert(!NameStrs.empty() && "No name data to emit");
442
443 uint8_t Header[16], *P = Header;
444 std::string UncompressedNameStrings =
445 join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
446
447 assert(StringRef(UncompressedNameStrings)
448 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
449 "PGO name is invalid (contains separator token)");
450
451 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
452 P += EncLen;
453
454 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
455 EncLen = encodeULEB128(CompressedLen, P);
456 P += EncLen;
457 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
458 unsigned HeaderLen = P - &Header[0];
459 Result.append(HeaderStr, HeaderLen);
460 Result += InputStr;
461 return Error::success();
462 };
463
464 if (!doCompression) {
465 return WriteStringToResult(0, UncompressedNameStrings);
466 }
467
468 SmallVector<uint8_t, 128> CompressedNameStrings;
469 compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings),
470 CompressedNameStrings,
472
473 return WriteStringToResult(CompressedNameStrings.size(),
474 toStringRef(CompressedNameStrings));
475}
476
478 auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
479 StringRef NameStr =
480 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
481 return NameStr;
482}
483
485 std::string &Result, bool doCompression) {
486 std::vector<std::string> NameStrs;
487 for (auto *NameVar : NameVars) {
488 NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
489 }
491 NameStrs, compression::zlib::isAvailable() && doCompression, Result);
492}
493
495 const uint8_t *P = NameStrings.bytes_begin();
496 const uint8_t *EndP = NameStrings.bytes_end();
497 while (P < EndP) {
498 uint32_t N;
499 uint64_t UncompressedSize = decodeULEB128(P, &N);
500 P += N;
501 uint64_t CompressedSize = decodeULEB128(P, &N);
502 P += N;
503 bool isCompressed = (CompressedSize != 0);
504 SmallVector<uint8_t, 128> UncompressedNameStrings;
505 StringRef NameStrings;
506 if (isCompressed) {
508 return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
509
510 if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
511 UncompressedNameStrings,
512 UncompressedSize)) {
513 consumeError(std::move(E));
514 return make_error<InstrProfError>(instrprof_error::uncompress_failed);
515 }
516 P += CompressedSize;
517 NameStrings = toStringRef(UncompressedNameStrings);
518 } else {
519 NameStrings =
520 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
521 P += UncompressedSize;
522 }
523 // Now parse the name strings.
525 NameStrings.split(Names, getInstrProfNameSeparator());
526 for (StringRef &Name : Names)
527 if (Error E = Symtab.addFuncName(Name))
528 return E;
529
530 while (P < EndP && *P == 0)
531 P++;
532 }
533 return Error::success();
534}
535
537 uint64_t FuncSum = 0;
538 Sum.NumEntries += Counts.size();
539 for (uint64_t Count : Counts)
540 FuncSum += Count;
541 Sum.CountSum += FuncSum;
542
543 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
544 uint64_t KindSum = 0;
545 uint32_t NumValueSites = getNumValueSites(VK);
546 for (size_t I = 0; I < NumValueSites; ++I) {
548 std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I);
549 for (uint32_t V = 0; V < NV; V++)
550 KindSum += VD[V].Count;
551 }
552 Sum.ValueCounts[VK] += KindSum;
553 }
554}
555
557 uint32_t ValueKind,
558 OverlapStats &Overlap,
559 OverlapStats &FuncLevelOverlap) {
560 this->sortByTargetValues();
561 Input.sortByTargetValues();
562 double Score = 0.0f, FuncLevelScore = 0.0f;
563 auto I = ValueData.begin();
564 auto IE = ValueData.end();
565 auto J = Input.ValueData.begin();
566 auto JE = Input.ValueData.end();
567 while (I != IE && J != JE) {
568 if (I->Value == J->Value) {
569 Score += OverlapStats::score(I->Count, J->Count,
570 Overlap.Base.ValueCounts[ValueKind],
571 Overlap.Test.ValueCounts[ValueKind]);
572 FuncLevelScore += OverlapStats::score(
573 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
574 FuncLevelOverlap.Test.ValueCounts[ValueKind]);
575 ++I;
576 } else if (I->Value < J->Value) {
577 ++I;
578 continue;
579 }
580 ++J;
581 }
582 Overlap.Overlap.ValueCounts[ValueKind] += Score;
583 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
584}
585
586// Return false on mismatch.
589 OverlapStats &Overlap,
590 OverlapStats &FuncLevelOverlap) {
591 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
592 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));
593 if (!ThisNumValueSites)
594 return;
595
596 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
597 getOrCreateValueSitesForKind(ValueKind);
599 Other.getValueSitesForKind(ValueKind);
600 for (uint32_t I = 0; I < ThisNumValueSites; I++)
601 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
602 FuncLevelOverlap);
603}
604
606 OverlapStats &FuncLevelOverlap,
607 uint64_t ValueCutoff) {
608 // FuncLevel CountSum for other should already computed and nonzero.
609 assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
610 accumulateCounts(FuncLevelOverlap.Base);
611 bool Mismatch = (Counts.size() != Other.Counts.size());
612
613 // Check if the value profiles mismatch.
614 if (!Mismatch) {
615 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
616 uint32_t ThisNumValueSites = getNumValueSites(Kind);
617 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
618 if (ThisNumValueSites != OtherNumValueSites) {
619 Mismatch = true;
620 break;
621 }
622 }
623 }
624 if (Mismatch) {
625 Overlap.addOneMismatch(FuncLevelOverlap.Test);
626 return;
627 }
628
629 // Compute overlap for value counts.
630 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
631 overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
632
633 double Score = 0.0;
634 uint64_t MaxCount = 0;
635 // Compute overlap for edge counts.
636 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
637 Score += OverlapStats::score(Counts[I], Other.Counts[I],
638 Overlap.Base.CountSum, Overlap.Test.CountSum);
639 MaxCount = std::max(Other.Counts[I], MaxCount);
640 }
641 Overlap.Overlap.CountSum += Score;
642 Overlap.Overlap.NumEntries += 1;
643
644 if (MaxCount >= ValueCutoff) {
645 double FuncScore = 0.0;
646 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
647 FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
648 FuncLevelOverlap.Base.CountSum,
649 FuncLevelOverlap.Test.CountSum);
650 FuncLevelOverlap.Overlap.CountSum = FuncScore;
651 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
652 FuncLevelOverlap.Valid = true;
653 }
654}
655
657 uint64_t Weight,
658 function_ref<void(instrprof_error)> Warn) {
659 this->sortByTargetValues();
660 Input.sortByTargetValues();
661 auto I = ValueData.begin();
662 auto IE = ValueData.end();
663 for (const InstrProfValueData &J : Input.ValueData) {
664 while (I != IE && I->Value < J.Value)
665 ++I;
666 if (I != IE && I->Value == J.Value) {
667 bool Overflowed;
668 I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed);
669 if (Overflowed)
671 ++I;
672 continue;
673 }
674 ValueData.insert(I, J);
675 }
676}
677
679 function_ref<void(instrprof_error)> Warn) {
680 for (InstrProfValueData &I : ValueData) {
681 bool Overflowed;
682 I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
683 if (Overflowed)
685 }
686}
687
688// Merge Value Profile data from Src record to this record for ValueKind.
689// Scale merged value counts by \p Weight.
690void InstrProfRecord::mergeValueProfData(
691 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
692 function_ref<void(instrprof_error)> Warn) {
693 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
694 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
695 if (ThisNumValueSites != OtherNumValueSites) {
697 return;
698 }
699 if (!ThisNumValueSites)
700 return;
701 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
702 getOrCreateValueSitesForKind(ValueKind);
704 Src.getValueSitesForKind(ValueKind);
705 for (uint32_t I = 0; I < ThisNumValueSites; I++)
706 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
707}
708
710 function_ref<void(instrprof_error)> Warn) {
711 // If the number of counters doesn't match we either have bad data
712 // or a hash collision.
713 if (Counts.size() != Other.Counts.size()) {
715 return;
716 }
717
718 // Special handling of the first count as the PseudoCount.
719 CountPseudoKind OtherKind = Other.getCountPseudoKind();
721 if (OtherKind != NotPseudo || ThisKind != NotPseudo) {
722 // We don't allow the merge of a profile with pseudo counts and
723 // a normal profile (i.e. without pesudo counts).
724 // Profile supplimenation should be done after the profile merge.
725 if (OtherKind == NotPseudo || ThisKind == NotPseudo) {
727 return;
728 }
729 if (OtherKind == PseudoHot || ThisKind == PseudoHot)
731 else
733 return;
734 }
735
736 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
737 bool Overflowed;
739 SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
742 Overflowed = true;
743 }
744 Counts[I] = Value;
745 if (Overflowed)
747 }
748
749 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
750 mergeValueProfData(Kind, Other, Weight, Warn);
751}
752
753void InstrProfRecord::scaleValueProfData(
754 uint32_t ValueKind, uint64_t N, uint64_t D,
755 function_ref<void(instrprof_error)> Warn) {
756 for (auto &R : getValueSitesForKind(ValueKind))
757 R.scale(N, D, Warn);
758}
759
761 function_ref<void(instrprof_error)> Warn) {
762 assert(D != 0 && "D cannot be 0");
763 for (auto &Count : this->Counts) {
764 bool Overflowed;
765 Count = SaturatingMultiply(Count, N, &Overflowed) / D;
766 if (Count > getInstrMaxCountValue()) {
767 Count = getInstrMaxCountValue();
768 Overflowed = true;
769 }
770 if (Overflowed)
772 }
773 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
774 scaleValueProfData(Kind, N, D, Warn);
775}
776
777// Map indirect call target name hash to name string.
778uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
779 InstrProfSymtab *SymTab) {
780 if (!SymTab)
781 return Value;
782
783 if (ValueKind == IPVK_IndirectCallTarget)
784 return SymTab->getFunctionHashFromAddress(Value);
785
786 return Value;
787}
788
790 InstrProfValueData *VData, uint32_t N,
792 for (uint32_t I = 0; I < N; I++) {
793 VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
794 }
795 std::vector<InstrProfValueSiteRecord> &ValueSites =
796 getOrCreateValueSitesForKind(ValueKind);
797 if (N == 0)
798 ValueSites.emplace_back();
799 else
800 ValueSites.emplace_back(VData, VData + N);
801}
802
803#define INSTR_PROF_COMMON_API_IMPL
805
806/*!
807 * ValueProfRecordClosure Interface implementation for InstrProfRecord
808 * class. These C wrappers are used as adaptors so that C++ code can be
809 * invoked as callbacks.
810 */
812 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
813}
814
816 return reinterpret_cast<const InstrProfRecord *>(Record)
817 ->getNumValueSites(VKind);
818}
819
821 return reinterpret_cast<const InstrProfRecord *>(Record)
822 ->getNumValueData(VKind);
823}
824
826 uint32_t S) {
827 return reinterpret_cast<const InstrProfRecord *>(R)
828 ->getNumValueDataForSite(VK, S);
829}
830
831void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
832 uint32_t K, uint32_t S) {
833 reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
834}
835
836ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
837 ValueProfData *VD =
838 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
839 memset(VD, 0, TotalSizeInBytes);
840 return VD;
841}
842
843static ValueProfRecordClosure InstrProfRecordClosure = {
844 nullptr,
849 nullptr,
852
853// Wrapper implementation using the closure mechanism.
854uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
855 auto Closure = InstrProfRecordClosure;
856 Closure.Record = &Record;
857 return getValueProfDataSize(&Closure);
858}
859
860// Wrapper implementation using the closure mechanism.
861std::unique_ptr<ValueProfData>
862ValueProfData::serializeFrom(const InstrProfRecord &Record) {
864
865 std::unique_ptr<ValueProfData> VPD(
866 serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
867 return VPD;
868}
869
870void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
871 InstrProfSymtab *SymTab) {
872 Record.reserveSites(Kind, NumValueSites);
873
874 InstrProfValueData *ValueData = getValueProfRecordValueData(this);
875 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
876 uint8_t ValueDataCount = this->SiteCountArray[VSite];
877 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
878 ValueData += ValueDataCount;
879 }
880}
881
882// For writing/serializing, Old is the host endianness, and New is
883// byte order intended on disk. For Reading/deserialization, Old
884// is the on-disk source endianness, and New is the host endianness.
885void ValueProfRecord::swapBytes(support::endianness Old,
887 using namespace support;
888
889 if (Old == New)
890 return;
891
892 if (getHostEndianness() != Old) {
893 sys::swapByteOrder<uint32_t>(NumValueSites);
894 sys::swapByteOrder<uint32_t>(Kind);
895 }
896 uint32_t ND = getValueProfRecordNumValueData(this);
897 InstrProfValueData *VD = getValueProfRecordValueData(this);
898
899 // No need to swap byte array: SiteCountArrray.
900 for (uint32_t I = 0; I < ND; I++) {
901 sys::swapByteOrder<uint64_t>(VD[I].Value);
902 sys::swapByteOrder<uint64_t>(VD[I].Count);
903 }
904 if (getHostEndianness() == Old) {
905 sys::swapByteOrder<uint32_t>(NumValueSites);
906 sys::swapByteOrder<uint32_t>(Kind);
907 }
908}
909
910void ValueProfData::deserializeTo(InstrProfRecord &Record,
911 InstrProfSymtab *SymTab) {
912 if (NumValueKinds == 0)
913 return;
914
915 ValueProfRecord *VR = getFirstValueProfRecord(this);
916 for (uint32_t K = 0; K < NumValueKinds; K++) {
917 VR->deserializeTo(Record, SymTab);
918 VR = getValueProfRecordNext(VR);
919 }
920}
921
922template <class T>
923static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
924 using namespace support;
925
926 if (Orig == little)
927 return endian::readNext<T, little, unaligned>(D);
928 else
929 return endian::readNext<T, big, unaligned>(D);
930}
931
932static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
933 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
934 ValueProfData());
935}
936
937Error ValueProfData::checkIntegrity() {
938 if (NumValueKinds > IPVK_Last + 1)
939 return make_error<InstrProfError>(
940 instrprof_error::malformed, "number of value profile kinds is invalid");
941 // Total size needs to be multiple of quadword size.
942 if (TotalSize % sizeof(uint64_t))
943 return make_error<InstrProfError>(
944 instrprof_error::malformed, "total size is not multiples of quardword");
945
946 ValueProfRecord *VR = getFirstValueProfRecord(this);
947 for (uint32_t K = 0; K < this->NumValueKinds; K++) {
948 if (VR->Kind > IPVK_Last)
949 return make_error<InstrProfError>(instrprof_error::malformed,
950 "value kind is invalid");
951 VR = getValueProfRecordNext(VR);
952 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
953 return make_error<InstrProfError>(
955 "value profile address is greater than total size");
956 }
957 return Error::success();
958}
959
961ValueProfData::getValueProfData(const unsigned char *D,
962 const unsigned char *const BufferEnd,
963 support::endianness Endianness) {
964 using namespace support;
965
966 if (D + sizeof(ValueProfData) > BufferEnd)
967 return make_error<InstrProfError>(instrprof_error::truncated);
968
969 const unsigned char *Header = D;
970 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
971 if (D + TotalSize > BufferEnd)
972 return make_error<InstrProfError>(instrprof_error::too_large);
973
974 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
975 memcpy(VPD.get(), D, TotalSize);
976 // Byte swap.
977 VPD->swapBytesToHost(Endianness);
978
979 Error E = VPD->checkIntegrity();
980 if (E)
981 return std::move(E);
982
983 return std::move(VPD);
984}
985
986void ValueProfData::swapBytesToHost(support::endianness Endianness) {
987 using namespace support;
988
989 if (Endianness == getHostEndianness())
990 return;
991
992 sys::swapByteOrder<uint32_t>(TotalSize);
993 sys::swapByteOrder<uint32_t>(NumValueKinds);
994
995 ValueProfRecord *VR = getFirstValueProfRecord(this);
996 for (uint32_t K = 0; K < NumValueKinds; K++) {
997 VR->swapBytes(Endianness, getHostEndianness());
998 VR = getValueProfRecordNext(VR);
999 }
1000}
1001
1002void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
1003 using namespace support;
1004
1005 if (Endianness == getHostEndianness())
1006 return;
1007
1008 ValueProfRecord *VR = getFirstValueProfRecord(this);
1009 for (uint32_t K = 0; K < NumValueKinds; K++) {
1010 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1011 VR->swapBytes(getHostEndianness(), Endianness);
1012 VR = NVR;
1013 }
1014 sys::swapByteOrder<uint32_t>(TotalSize);
1015 sys::swapByteOrder<uint32_t>(NumValueKinds);
1016}
1017
1019 const InstrProfRecord &InstrProfR,
1020 InstrProfValueKind ValueKind, uint32_t SiteIdx,
1021 uint32_t MaxMDCount) {
1022 uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
1023 if (!NV)
1024 return;
1025
1026 uint64_t Sum = 0;
1027 std::unique_ptr<InstrProfValueData[]> VD =
1028 InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
1029
1030 ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
1031 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1032}
1033
1036 uint64_t Sum, InstrProfValueKind ValueKind,
1037 uint32_t MaxMDCount) {
1038 LLVMContext &Ctx = M.getContext();
1039 MDBuilder MDHelper(Ctx);
1041 // Tag
1042 Vals.push_back(MDHelper.createString("VP"));
1043 // Value Kind
1044 Vals.push_back(MDHelper.createConstant(
1045 ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
1046 // Total Count
1047 Vals.push_back(
1048 MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
1049
1050 // Value Profile Data
1051 uint32_t MDCount = MaxMDCount;
1052 for (auto &VD : VDs) {
1053 Vals.push_back(MDHelper.createConstant(
1054 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
1055 Vals.push_back(MDHelper.createConstant(
1056 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
1057 if (--MDCount == 0)
1058 break;
1059 }
1060 Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
1061}
1062
1064 InstrProfValueKind ValueKind,
1065 uint32_t MaxNumValueData,
1066 InstrProfValueData ValueData[],
1067 uint32_t &ActualNumValueData, uint64_t &TotalC,
1068 bool GetNoICPValue) {
1069 MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1070 if (!MD)
1071 return false;
1072
1073 unsigned NOps = MD->getNumOperands();
1074
1075 if (NOps < 5)
1076 return false;
1077
1078 // Operand 0 is a string tag "VP":
1079 MDString *Tag = cast<MDString>(MD->getOperand(0));
1080 if (!Tag)
1081 return false;
1082
1083 if (!Tag->getString().equals("VP"))
1084 return false;
1085
1086 // Now check kind:
1087 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1088 if (!KindInt)
1089 return false;
1090 if (KindInt->getZExtValue() != ValueKind)
1091 return false;
1092
1093 // Get total count
1094 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1095 if (!TotalCInt)
1096 return false;
1097 TotalC = TotalCInt->getZExtValue();
1098
1099 ActualNumValueData = 0;
1100
1101 for (unsigned I = 3; I < NOps; I += 2) {
1102 if (ActualNumValueData >= MaxNumValueData)
1103 break;
1104 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1105 ConstantInt *Count =
1106 mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1107 if (!Value || !Count)
1108 return false;
1109 uint64_t CntValue = Count->getZExtValue();
1110 if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1111 continue;
1112 ValueData[ActualNumValueData].Value = Value->getZExtValue();
1113 ValueData[ActualNumValueData].Count = CntValue;
1114 ActualNumValueData++;
1115 }
1116 return true;
1117}
1118
1120 return F.getMetadata(getPGOFuncNameMetadataName());
1121}
1122
1124 // Only for internal linkage functions.
1125 if (PGOFuncName == F.getName())
1126 return;
1127 // Don't create duplicated meta-data.
1129 return;
1130 LLVMContext &C = F.getContext();
1131 MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1132 F.setMetadata(getPGOFuncNameMetadataName(), N);
1133}
1134
1135bool needsComdatForCounter(const Function &F, const Module &M) {
1136 if (F.hasComdat())
1137 return true;
1138
1139 if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1140 return false;
1141
1142 // See createPGOFuncNameVar for more details. To avoid link errors, profile
1143 // counters for function with available_externally linkage needs to be changed
1144 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1145 // created. Without using comdat, duplicate entries won't be removed by the
1146 // linker leading to increased data segement size and raw profile size. Even
1147 // worse, since the referenced counter from profile per-function data object
1148 // will be resolved to the common strong definition, the profile counts for
1149 // available_externally functions will end up being duplicated in raw profile
1150 // data. This can result in distorted profile as the counts of those dups
1151 // will be accumulated by the profile merger.
1152 GlobalValue::LinkageTypes Linkage = F.getLinkage();
1153 if (Linkage != GlobalValue::ExternalWeakLinkage &&
1155 return false;
1156
1157 return true;
1158}
1159
1160// Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1161bool isIRPGOFlagSet(const Module *M) {
1162 auto IRInstrVar =
1163 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1164 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1165 return false;
1166
1167 // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1168 // have the decl.
1169 if (IRInstrVar->isDeclaration())
1170 return true;
1171
1172 // Check if the flag is set.
1173 if (!IRInstrVar->hasInitializer())
1174 return false;
1175
1176 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1177 if (!InitVal)
1178 return false;
1179 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1180}
1181
1182// Check if we can safely rename this Comdat function.
1183bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1184 if (F.getName().empty())
1185 return false;
1186 if (!needsComdatForCounter(F, *(F.getParent())))
1187 return false;
1188 // Unsafe to rename the address-taken function (which can be used in
1189 // function comparison).
1190 if (CheckAddressTaken && F.hasAddressTaken())
1191 return false;
1192 // Only safe to do if this function may be discarded if it is not used
1193 // in the compilation unit.
1194 if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1195 return false;
1196
1197 // For AvailableExternallyLinkage functions.
1198 if (!F.hasComdat()) {
1200 return true;
1201 }
1202 return true;
1203}
1204
1205// Create the variable for the profile file name.
1206void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1207 if (InstrProfileOutput.empty())
1208 return;
1209 Constant *ProfileNameConst =
1210 ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1211 GlobalVariable *ProfileNameVar = new GlobalVariable(
1212 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1213 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1215 Triple TT(M.getTargetTriple());
1216 if (TT.supportsCOMDAT()) {
1218 ProfileNameVar->setComdat(M.getOrInsertComdat(
1219 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1220 }
1221}
1222
1223Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1224 const std::string &TestFilename,
1225 bool IsCS) {
1226 auto getProfileSum = [IsCS](const std::string &Filename,
1227 CountSumOrPercent &Sum) -> Error {
1228 // This function is only used from llvm-profdata that doesn't use any kind
1229 // of VFS. Just create a default RealFileSystem to read profiles.
1230 auto FS = vfs::getRealFileSystem();
1231 auto ReaderOrErr = InstrProfReader::create(Filename, *FS);
1232 if (Error E = ReaderOrErr.takeError()) {
1233 return E;
1234 }
1235 auto Reader = std::move(ReaderOrErr.get());
1236 Reader->accumulateCounts(Sum, IsCS);
1237 return Error::success();
1238 };
1239 auto Ret = getProfileSum(BaseFilename, Base);
1240 if (Ret)
1241 return Ret;
1242 Ret = getProfileSum(TestFilename, Test);
1243 if (Ret)
1244 return Ret;
1245 this->BaseFilename = &BaseFilename;
1246 this->TestFilename = &TestFilename;
1247 Valid = true;
1248 return Error::success();
1249}
1250
1252 Mismatch.NumEntries += 1;
1253 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1254 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1255 if (Test.ValueCounts[I] >= 1.0f)
1257 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1258 }
1259}
1260
1262 Unique.NumEntries += 1;
1263 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1264 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1265 if (Test.ValueCounts[I] >= 1.0f)
1266 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1267 }
1268}
1269
1271 if (!Valid)
1272 return;
1273
1274 const char *EntryName =
1275 (Level == ProgramLevel ? "functions" : "edge counters");
1276 if (Level == ProgramLevel) {
1277 OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1278 << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1279 } else {
1280 OS << "Function level:\n"
1281 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1282 }
1283
1284 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1285 if (Mismatch.NumEntries)
1286 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1287 << "\n";
1288 if (Unique.NumEntries)
1289 OS << " # of " << EntryName
1290 << " only in test_profile: " << Unique.NumEntries << "\n";
1291
1292 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1293 << "\n";
1294 if (Mismatch.NumEntries)
1295 OS << " Mismatched count percentage (Edge): "
1296 << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1297 if (Unique.NumEntries)
1298 OS << " Percentage of Edge profile only in test_profile: "
1299 << format("%.3f%%", Unique.CountSum * 100) << "\n";
1300 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1301 << "\n"
1302 << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1303 << "\n";
1304
1305 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1306 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1307 continue;
1308 char ProfileKindName[20];
1309 switch (I) {
1310 case IPVK_IndirectCallTarget:
1311 strncpy(ProfileKindName, "IndirectCall", 19);
1312 break;
1313 case IPVK_MemOPSize:
1314 strncpy(ProfileKindName, "MemOP", 19);
1315 break;
1316 default:
1317 snprintf(ProfileKindName, 19, "VP[%d]", I);
1318 break;
1319 }
1320 OS << " " << ProfileKindName
1321 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1322 << "\n";
1323 if (Mismatch.NumEntries)
1324 OS << " Mismatched count percentage (" << ProfileKindName
1325 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1326 if (Unique.NumEntries)
1327 OS << " Percentage of " << ProfileKindName
1328 << " profile only in test_profile: "
1329 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1330 OS << " " << ProfileKindName
1331 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1332 << "\n"
1333 << " " << ProfileKindName
1334 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1335 << "\n";
1336 }
1337}
1338
1339namespace IndexedInstrProf {
1340// A C++14 compatible version of the offsetof macro.
1341template <typename T1, typename T2>
1342inline size_t constexpr offsetOf(T1 T2::*Member) {
1343 constexpr T2 Object{};
1344 return size_t(&(Object.*Member)) - size_t(&Object);
1345}
1346
1347static inline uint64_t read(const unsigned char *Buffer, size_t Offset) {
1348 return *reinterpret_cast<const uint64_t *>(Buffer + Offset);
1349}
1350
1352 using namespace support;
1353 return endian::byte_swap<uint64_t, little>(Version);
1354}
1355
1356Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
1357 using namespace support;
1358 static_assert(std::is_standard_layout_v<Header>,
1359 "The header should be standard layout type since we use offset "
1360 "of fields to read.");
1361 Header H;
1362
1363 H.Magic = read(Buffer, offsetOf(&Header::Magic));
1364 // Check the magic number.
1365 uint64_t Magic = endian::byte_swap<uint64_t, little>(H.Magic);
1367 return make_error<InstrProfError>(instrprof_error::bad_magic);
1368
1369 // Read the version.
1370 H.Version = read(Buffer, offsetOf(&Header::Version));
1371 if (GET_VERSION(H.formatVersion()) >
1373 return make_error<InstrProfError>(instrprof_error::unsupported_version);
1374
1375 switch (GET_VERSION(H.formatVersion())) {
1376 // When a new field is added in the header add a case statement here to
1377 // populate it.
1378 static_assert(
1380 "Please update the reading code below if a new field has been added, "
1381 "if not add a case statement to fall through to the latest version.");
1382 case 9ull:
1383 H.BinaryIdOffset = read(Buffer, offsetOf(&Header::BinaryIdOffset));
1384 [[fallthrough]];
1385 case 8ull:
1386 H.MemProfOffset = read(Buffer, offsetOf(&Header::MemProfOffset));
1387 [[fallthrough]];
1388 default: // Version7 (when the backwards compatible header was introduced).
1389 H.HashType = read(Buffer, offsetOf(&Header::HashType));
1390 H.HashOffset = read(Buffer, offsetOf(&Header::HashOffset));
1391 }
1392
1393 return H;
1394}
1395
1396size_t Header::size() const {
1397 switch (GET_VERSION(formatVersion())) {
1398 // When a new field is added to the header add a case statement here to
1399 // compute the size as offset of the new field + size of the new field. This
1400 // relies on the field being added to the end of the list.
1402 "Please update the size computation below if a new field has "
1403 "been added to the header, if not add a case statement to "
1404 "fall through to the latest version.");
1405 case 9ull:
1407 case 8ull:
1409 default: // Version7 (when the backwards compatible header was introduced).
1411 }
1412}
1413
1414} // namespace IndexedInstrProf
1415
1416} // end namespace llvm
aarch64 promote const
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_UNUSED
Definition: Compiler.h:172
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::string Name
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Definition: InstrProf.cpp:78
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define H(x, y, z)
Definition: MD5.cpp:57
This file contains the declarations for metadata subclasses.
Module.h This file contains the declarations for the Module class.
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const char * name
Definition: SMEABIPass.cpp:49
raw_pwrite_stream & OS
This file defines the SmallString class.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
@ Names
Definition: TextStubV5.cpp:106
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:152
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
iterator begin() const
Definition: ArrayRef.h:151
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:158
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2946
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:145
This is an important base class in LLVM.
Definition: Constant.h:41
Lightweight error class with error context and mandatory checking.
Definition: Error.h:156
static ErrorSuccess success()
Create a success value.
Definition: Error.h:330
Tagged union holding either a T or a Error.
Definition: Error.h:470
void setComdat(Comdat *C)
Definition: Globals.cpp:198
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:404
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:591
bool isDiscardableIfUnused() const
Definition: GlobalValue.h:543
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:250
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:170
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
Definition: GlobalValue.h:50
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
@ ExternalWeakLinkage
ExternalWeak linkage description.
Definition: GlobalValue.h:57
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
static char ID
Definition: InstrProf.h:367
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:257
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:446
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:425
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:507
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:275
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1455
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
Definition: MDBuilder.cpp:24
MDString * createString(StringRef Str)
Return the given string as metadata.
Definition: MDBuilder.cpp:20
Metadata node.
Definition: Metadata.h:943
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1291
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1399
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1297
A single uniqued string.
Definition: Metadata.h:611
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:497
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:305
size_t size() const
Definition: SmallVector.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
Definition: InstrProf.cpp:232
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:687
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:222
const unsigned char * bytes_end() const
Definition: StringRef.h:118
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:558
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition: StringRef.h:596
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
bool startswith(StringRef Prefix) const
Definition: StringRef.h:261
const unsigned char * bytes_begin() const
Definition: StringRef.h:115
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:976
ObjectFormatType
Definition: Triple.h:280
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
See the file comment.
Definition: ValueMap.h:84
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
static uint64_t read(const unsigned char *Buffer, size_t Offset)
Definition: InstrProf.cpp:1347
const uint64_t Magic
Definition: InstrProf.h:1025
size_t constexpr offsetOf(T1 T2::*Member)
Definition: InstrProf.cpp:1342
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
Definition: Compression.h:39
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
Definition: Path.cpp:601
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1063
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:90
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:612
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
Definition: InstrProf.cpp:494
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Definition: InstrProf.cpp:301
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1123
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:272
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
Definition: InstrProf.cpp:831
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:324
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
Definition: STLExtras.h:2076
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
Definition: LEB128.h:128
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1119
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
Definition: InstrProf.cpp:932
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig)
Definition: InstrProf.cpp:923
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:213
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition: InstrProf.h:64
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:380
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1018
InstrProfSectKind
Definition: InstrProf.h:58
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition: MathExtras.h:658
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:477
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:169
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:124
support::endianness getHostEndianness()
Definition: InstrProf.h:991
instrprof_error
Definition: InstrProf.h:308
InstrProfValueKind
Definition: InstrProf.h:244
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:180
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:2011
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1135
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:815
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1183
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1206
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:439
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
Definition: InstrProf.cpp:825
static ValueProfRecordClosure InstrProfRecordClosure
Definition: InstrProf.cpp:843
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:335
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
Definition: InstrProf.cpp:273
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1161
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1043
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
Definition: Metadata.h:56
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
Definition: InstrProf.cpp:811
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
Definition: InstrProf.cpp:836
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:820
#define N
double ValueCounts[IPVK_Last - IPVK_First+1]
Definition: InstrProf.h:639
uint64_t formatVersion() const
Definition: InstrProf.cpp:1351
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1356
Profiling information for a single function.
Definition: InstrProf.h:730
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:587
std::vector< uint64_t > Counts
Definition: InstrProf.h:731
CountPseudoKind getCountPseudoKind() const
Definition: InstrProf.h:836
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:536
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:945
void setPseudoCount(CountPseudoKind Kind)
Definition: InstrProf.h:844
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:709
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:789
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:949
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:605
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
Definition: InstrProf.h:955
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:760
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:708
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:656
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:556
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:700
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:678
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1251
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:683
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1223
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1270
CountSumOrPercent Overlap
Definition: InstrProf.h:657
CountSumOrPercent Base
Definition: InstrProf.h:653
uint64_t FuncHash
Definition: InstrProf.h:664
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1261
const std::string * BaseFilename
Definition: InstrProf.h:661
const std::string * TestFilename
Definition: InstrProf.h:662
CountSumOrPercent Unique
Definition: InstrProf.h:659
CountSumOrPercent Mismatch
Definition: InstrProf.h:658
StringRef FuncName
Definition: InstrProf.h:663
OverlapStatsLevel Level
Definition: InstrProf.h:660
CountSumOrPercent Test
Definition: InstrProf.h:655