LLVM 17.0.0git
InstrProf.h
Go to the documentation of this file.
1//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Instrumentation-based profiling data is generated by instrumented
10// binaries through library functions in compiler-rt, and read by the clang
11// frontend to feed PGO.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
17
18#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/StringSet.h"
23#include "llvm/IR/GlobalValue.h"
28#include "llvm/Support/Endian.h"
29#include "llvm/Support/Error.h"
31#include "llvm/Support/MD5.h"
36#include <algorithm>
37#include <cassert>
38#include <cstddef>
39#include <cstdint>
40#include <cstring>
41#include <list>
42#include <memory>
43#include <string>
44#include <system_error>
45#include <utility>
46#include <vector>
47
48namespace llvm {
49
50class Function;
51class GlobalVariable;
52struct InstrProfRecord;
53class InstrProfSymtab;
54class Instruction;
55class MDNode;
56class Module;
57
59#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
61};
62
63/// Return the max count value. We reserver a few large values for special use.
65 return std::numeric_limits<uint64_t>::max() - 2;
66}
67
68/// Return the name of the profile section corresponding to \p IPSK.
69///
70/// The name of the section depends on the object format type \p OF. If
71/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
72/// be added to the section name (this is the default).
75 bool AddSegmentInfo = true);
76
77/// Return the name profile runtime entry point to do value profiling
78/// for a given site.
80 return INSTR_PROF_VALUE_PROF_FUNC_STR;
81}
82
83/// Return the name profile runtime entry point to do memop size value
84/// profiling.
86 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
87}
88
89/// Return the name prefix of variables containing instrumented function names.
90inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
91
92/// Return the name prefix of variables containing per-function control data.
93inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
94
95/// Return the name prefix of profile counter variables.
96inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
97
98/// Return the name prefix of value profile variables.
99inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
100
101/// Return the name of value profile node array variables:
102inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
103
104/// Return the name of the variable holding the strings (possibly compressed)
105/// of all function's PGO names.
107 return "__llvm_prf_nm";
108}
109
110/// Return the name of a covarage mapping variable (internal linkage)
111/// for each instrumented source module. Such variables are allocated
112/// in the __llvm_covmap section.
114 return "__llvm_coverage_mapping";
115}
116
117/// Return the name of the internal variable recording the array
118/// of PGO name vars referenced by the coverage mapping. The owning
119/// functions of those names are not emitted by FE (e.g, unused inline
120/// functions.)
122 return "__llvm_coverage_names";
123}
124
125/// Return the name of function that registers all the per-function control
126/// data at program startup time by calling __llvm_register_function. This
127/// function has internal linkage and is called by __llvm_profile_init
128/// runtime method. This function is not generated for these platforms:
129/// Darwin, Linux, and FreeBSD.
131 return "__llvm_profile_register_functions";
132}
133
134/// Return the name of the runtime interface that registers per-function control
135/// data for one instrumented function.
137 return "__llvm_profile_register_function";
138}
139
140/// Return the name of the runtime interface that registers the PGO name strings.
142 return "__llvm_profile_register_names_function";
143}
144
145/// Return the name of the runtime initialization method that is generated by
146/// the compiler. The function calls __llvm_profile_register_functions and
147/// __llvm_profile_override_default_filename functions if needed. This function
148/// has internal linkage and invoked at startup time via init_array.
149inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
150
151/// Return the name of the hook variable defined in profile runtime library.
152/// A reference to the variable causes the linker to link in the runtime
153/// initialization module (which defines the hook variable).
155 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
156}
157
158/// Return the name of the compiler generated function that references the
159/// runtime hook variable. The function is a weak global.
161 return "__llvm_profile_runtime_user";
162}
163
165 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
166}
167
168/// Return the marker used to separate PGO names during serialization.
169inline StringRef getInstrProfNameSeparator() { return "\01"; }
170
171/// Return the modified name for function \c F suitable to be
172/// used the key for profile lookup. Variable \c InLTO indicates if this
173/// is called in LTO optimization passes.
174std::string getPGOFuncName(const Function &F, bool InLTO = false,
175 uint64_t Version = INSTR_PROF_INDEX_VERSION);
176
177/// Return the modified name for a function suitable to be
178/// used the key for profile lookup. The function's original
179/// name is \c RawFuncName and has linkage of type \c Linkage.
180/// The function is defined in module \c FileName.
181std::string getPGOFuncName(StringRef RawFuncName,
183 StringRef FileName,
184 uint64_t Version = INSTR_PROF_INDEX_VERSION);
185
186/// Return the name of the global variable used to store a function
187/// name in PGO instrumentation. \c FuncName is the name of the function
188/// returned by the \c getPGOFuncName call.
189std::string getPGOFuncNameVarName(StringRef FuncName,
191
192/// Create and return the global variable for function name used in PGO
193/// instrumentation. \c FuncName is the name of the function returned
194/// by \c getPGOFuncName call.
195GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
196
197/// Create and return the global variable for function name used in PGO
198/// instrumentation. /// \c FuncName is the name of the function
199/// returned by \c getPGOFuncName call, \c M is the owning module,
200/// and \c Linkage is the linkage of the instrumented function.
201GlobalVariable *createPGOFuncNameVar(Module &M,
203 StringRef PGOFuncName);
204
205/// Return the initializer in string of the PGO name var \c NameVar.
206StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
207
208/// Given a PGO function name, remove the filename prefix and return
209/// the original (static) function name.
210StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
211 StringRef FileName = "<unknown>");
212
213/// Given a vector of strings (function PGO names) \c NameStrs, the
214/// method generates a combined string \c Result that is ready to be
215/// serialized. The \c Result string is comprised of three fields:
216/// The first field is the length of the uncompressed strings, and the
217/// the second field is the length of the zlib-compressed string.
218/// Both fields are encoded in ULEB128. If \c doCompress is false, the
219/// third field is the uncompressed strings; otherwise it is the
220/// compressed string. When the string compression is off, the
221/// second field will have value zero.
222Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
223 bool doCompression, std::string &Result);
224
225/// Produce \c Result string with the same format described above. The input
226/// is vector of PGO function name variables that are referenced.
227Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
228 std::string &Result, bool doCompression = true);
229
230/// \c NameStrings is a string composed of one of more sub-strings encoded in
231/// the format described above. The substrings are separated by 0 or more zero
232/// bytes. This method decodes the string and populates the \c Symtab.
233Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
234
235/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
236/// set in IR PGO compilation.
237bool isIRPGOFlagSet(const Module *M);
238
239/// Check if we can safely rename this Comdat function. Instances of the same
240/// comdat function may have different control flows thus can not share the
241/// same counter variable.
242bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
243
245#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
247};
248
249/// Get the value profile data for value site \p SiteIdx from \p InstrProfR
250/// and annotate the instruction \p Inst with the value profile meta data.
251/// Annotate up to \p MaxMDCount (default 3) number of records per value site.
252void annotateValueSite(Module &M, Instruction &Inst,
253 const InstrProfRecord &InstrProfR,
254 InstrProfValueKind ValueKind, uint32_t SiteIndx,
255 uint32_t MaxMDCount = 3);
256
257/// Same as the above interface but using an ArrayRef, as well as \p Sum.
258void annotateValueSite(Module &M, Instruction &Inst,
259 ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
260 InstrProfValueKind ValueKind, uint32_t MaxMDCount);
261
262/// Extract the value profile data from \p Inst which is annotated with
263/// value profile meta data. Return false if there is no value data annotated,
264/// otherwise return true.
265bool getValueProfDataFromInst(const Instruction &Inst,
266 InstrProfValueKind ValueKind,
267 uint32_t MaxNumValueData,
268 InstrProfValueData ValueData[],
269 uint32_t &ActualNumValueData, uint64_t &TotalC,
270 bool GetNoICPValue = false);
271
272inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
273
274/// Return the PGOFuncName meta data associated with a function.
275MDNode *getPGOFuncNameMetadata(const Function &F);
276
277/// Create the PGOFuncName meta data if PGOFuncName is different from
278/// function's raw name. This should only apply to internal linkage functions
279/// declared by users only.
280void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
281
282/// Check if we can use Comdat for profile variables. This will eliminate
283/// the duplicated profile variables for Comdat functions.
284bool needsComdatForCounter(const Function &F, const Module &M);
285
286/// An enum describing the attributes of an instrumented profile.
287enum class InstrProfKind {
288 Unknown = 0x0,
289 // A frontend clang profile, incompatible with other attrs.
291 // An IR-level profile (default when -fprofile-generate is used).
292 IRInstrumentation = 0x2,
293 // A profile with entry basic block instrumentation.
295 // A context sensitive IR-level profile.
296 ContextSensitive = 0x8,
297 // Use single byte probes for coverage.
298 SingleByteCoverage = 0x10,
299 // Only instrument the function entry basic block.
300 FunctionEntryOnly = 0x20,
301 // A memory profile collected using -fprofile=memory.
302 MemProf = 0x40,
303 // A temporal profile.
304 TemporalProfile = 0x80,
306};
307
308const std::error_category &instrprof_category();
309
310enum class instrprof_error {
311 success = 0,
312 eof,
314 bad_magic,
318 too_large,
319 truncated,
320 malformed,
335};
336
337/// An ordered list of functions identified by their NameRef found in
338/// INSTR_PROF_DATA
341 std::vector<uint64_t> FunctionNameRefs;
342};
343
344inline std::error_code make_error_code(instrprof_error E) {
345 return std::error_code(static_cast<int>(E), instrprof_category());
346}
347
348class InstrProfError : public ErrorInfo<InstrProfError> {
349public:
351 : Err(Err), Msg(ErrStr.str()) {
352 assert(Err != instrprof_error::success && "Not an error");
353 }
354
355 std::string message() const override;
356
357 void log(raw_ostream &OS) const override { OS << message(); }
358
359 std::error_code convertToErrorCode() const override {
360 return make_error_code(Err);
361 }
362
363 instrprof_error get() const { return Err; }
364 const std::string &getMessage() const { return Msg; }
365
366 /// Consume an Error and return the raw enum value contained within it, and
367 /// the optional error message. The Error must either be a success value, or
368 /// contain a single InstrProfError.
369 static std::pair<instrprof_error, std::string> take(Error E) {
370 auto Err = instrprof_error::success;
371 std::string Msg = "";
372 handleAllErrors(std::move(E), [&Err, &Msg](const InstrProfError &IPE) {
373 assert(Err == instrprof_error::success && "Multiple errors encountered");
374 Err = IPE.get();
375 Msg = IPE.getMessage();
376 });
377 return {Err, Msg};
378 }
379
380 static char ID;
381
382private:
383 instrprof_error Err;
384 std::string Msg;
385};
386
388 /// Count the number of soft instrprof_errors encountered and keep track of
389 /// the first such error for reporting purposes.
390
391 /// The first soft error encountered.
393
394 /// The number of hash mismatches.
395 unsigned NumHashMismatches = 0;
396
397 /// The number of count mismatches.
398 unsigned NumCountMismatches = 0;
399
400 /// The number of counter overflows.
401 unsigned NumCounterOverflows = 0;
402
403 /// The number of value site count mismatches.
404 unsigned NumValueSiteCountMismatches = 0;
405
406public:
408
410 assert(FirstError == instrprof_error::success &&
411 "Unchecked soft error encountered");
412 }
413
414 /// Track a soft error (\p IE) and increment its associated counter.
415 void addError(instrprof_error IE);
416
417 /// Get the number of hash mismatches.
418 unsigned getNumHashMismatches() const { return NumHashMismatches; }
419
420 /// Get the number of count mismatches.
421 unsigned getNumCountMismatches() const { return NumCountMismatches; }
422
423 /// Get the number of counter overflows.
424 unsigned getNumCounterOverflows() const { return NumCounterOverflows; }
425
426 /// Get the number of value site count mismatches.
428 return NumValueSiteCountMismatches;
429 }
430
431 /// Return the first encountered error and reset FirstError to a success
432 /// value.
434 if (FirstError == instrprof_error::success)
435 return Error::success();
436 auto E = make_error<InstrProfError>(FirstError);
437 FirstError = instrprof_error::success;
438 return E;
439 }
440};
441
442namespace object {
443
444class SectionRef;
445
446} // end namespace object
447
448namespace IndexedInstrProf {
449
451
452} // end namespace IndexedInstrProf
453
454/// A symbol table used for function PGO name look-up with keys
455/// (such as pointers, md5hash values) to the function. A function's
456/// PGO name or name's md5hash are used in retrieving the profile
457/// data of the function. See \c getPGOFuncName() method for details
458/// on how PGO name is formed.
460public:
461 using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
462
463private:
464 StringRef Data;
465 uint64_t Address = 0;
466 // Unique name strings.
467 StringSet<> NameTab;
468 // A map from MD5 keys to function name strings.
469 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
470 // A map from MD5 keys to function define. We only populate this map
471 // when build the Symtab from a Module.
472 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
473 // A map from function runtime address to function name MD5 hash.
474 // This map is only populated and used by raw instr profile reader.
475 AddrHashMap AddrToMD5Map;
476 bool Sorted = false;
477
478 static StringRef getExternalSymbol() {
479 return "** External Symbol **";
480 }
481
482 // If the symtab is created by a series of calls to \c addFuncName, \c
483 // finalizeSymtab needs to be called before looking up function names.
484 // This is required because the underlying map is a vector (for space
485 // efficiency) which needs to be sorted.
486 inline void finalizeSymtab();
487
488public:
489 InstrProfSymtab() = default;
490
491 /// Create InstrProfSymtab from an object file section which
492 /// contains function PGO names. When section may contain raw
493 /// string data or string data in compressed form. This method
494 /// only initialize the symtab with reference to the data and
495 /// the section base address. The decompression will be delayed
496 /// until before it is used. See also \c create(StringRef) method.
498
499 /// This interface is used by reader of CoverageMapping test
500 /// format.
501 inline Error create(StringRef D, uint64_t BaseAddr);
502
503 /// \c NameStrings is a string composed of one of more sub-strings
504 /// encoded in the format described in \c collectPGOFuncNameStrings.
505 /// This method is a wrapper to \c readPGOFuncNameStrings method.
506 inline Error create(StringRef NameStrings);
507
508 /// A wrapper interface to populate the PGO symtab with functions
509 /// decls from module \c M. This interface is used by transformation
510 /// passes such as indirect function call promotion. Variable \c InLTO
511 /// indicates if this is called from LTO optimization passes.
512 Error create(Module &M, bool InLTO = false);
513
514 /// Create InstrProfSymtab from a set of names iteratable from
515 /// \p IterRange. This interface is used by IndexedProfReader.
516 template <typename NameIterRange> Error create(const NameIterRange &IterRange);
517
518 /// Update the symtab by adding \p FuncName to the table. This interface
519 /// is used by the raw and text profile readers.
521 if (FuncName.empty())
522 return make_error<InstrProfError>(instrprof_error::malformed,
523 "function name is empty");
524 auto Ins = NameTab.insert(FuncName);
525 if (Ins.second) {
526 MD5NameMap.push_back(std::make_pair(
527 IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
528 Sorted = false;
529 }
530 return Error::success();
531 }
532
533 /// Map a function address to its name's MD5 hash. This interface
534 /// is only used by the raw profiler reader.
536 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
537 }
538
539 /// Return a function's hash, or 0, if the function isn't in this SymTab.
541
542 /// Return function's PGO name from the function name's symbol
543 /// address in the object file. If an error occurs, return
544 /// an empty string.
545 StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
546
547 /// Return function's PGO name from the name's md5 hash value.
548 /// If not found, return an empty string.
549 inline StringRef getFuncName(uint64_t FuncMD5Hash);
550
551 /// Just like getFuncName, except that it will return a non-empty StringRef
552 /// if the function is external to this symbol table. All such cases
553 /// will be represented using the same StringRef value.
555
556 /// True if Symbol is the value used to represent external symbols.
557 static bool isExternalSymbol(const StringRef &Symbol) {
558 return Symbol == InstrProfSymtab::getExternalSymbol();
559 }
560
561 /// Return function from the name's md5 hash. Return nullptr if not found.
562 inline Function *getFunction(uint64_t FuncMD5Hash);
563
564 /// Return the function's original assembly name by stripping off
565 /// the prefix attached (to symbols with priviate linkage). For
566 /// global functions, it returns the same string as getFuncName.
567 inline StringRef getOrigFuncName(uint64_t FuncMD5Hash);
568
569 /// Return the name section data.
570 inline StringRef getNameData() const { return Data; }
571
572 /// Dump the symbols in this table.
573 void dumpNames(raw_ostream &OS) const {
574 for (StringRef S : NameTab.keys())
575 OS << S << "\n";
576 }
577};
578
580 Data = D;
581 Address = BaseAddr;
582 return Error::success();
583}
584
586 return readPGOFuncNameStrings(NameStrings, *this);
587}
588
589template <typename NameIterRange>
590Error InstrProfSymtab::create(const NameIterRange &IterRange) {
591 for (auto Name : IterRange)
592 if (Error E = addFuncName(Name))
593 return E;
594
595 finalizeSymtab();
596 return Error::success();
597}
598
599void InstrProfSymtab::finalizeSymtab() {
600 if (Sorted)
601 return;
602 llvm::sort(MD5NameMap, less_first());
603 llvm::sort(MD5FuncMap, less_first());
604 llvm::sort(AddrToMD5Map, less_first());
605 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
606 AddrToMD5Map.end());
607 Sorted = true;
608}
609
611 StringRef ret = getFuncName(FuncMD5Hash);
612 if (ret.empty())
613 return InstrProfSymtab::getExternalSymbol();
614 return ret;
615}
616
618 finalizeSymtab();
619 auto Result = llvm::lower_bound(MD5NameMap, FuncMD5Hash,
620 [](const std::pair<uint64_t, StringRef> &LHS,
621 uint64_t RHS) { return LHS.first < RHS; });
622 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
623 return Result->second;
624 return StringRef();
625}
626
628 finalizeSymtab();
629 auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
630 [](const std::pair<uint64_t, Function *> &LHS,
631 uint64_t RHS) { return LHS.first < RHS; });
632 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
633 return Result->second;
634 return nullptr;
635}
636
637// See also getPGOFuncName implementation. These two need to be
638// matched.
640 StringRef PGOName = getFuncName(FuncMD5Hash);
641 size_t S = PGOName.find_first_of(':');
642 if (S == StringRef::npos)
643 return PGOName;
644 return PGOName.drop_front(S + 1);
645}
646
647// To store the sums of profile count values, or the percentage of
648// the sums of the total count values.
651 double CountSum;
652 double ValueCounts[IPVK_Last - IPVK_First + 1];
654 void reset() {
655 NumEntries = 0;
656 CountSum = 0.0f;
657 for (double &VC : ValueCounts)
658 VC = 0.0f;
659 }
660};
661
662// Function level or program level overlap information.
665 // Sum of the total count values for the base profile.
667 // Sum of the total count values for the test profile.
669 // Overlap lap score. Should be in range of [0.0f to 1.0f].
674 const std::string *BaseFilename;
675 const std::string *TestFilename;
678 bool Valid;
679
681 : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0),
682 Valid(false) {}
683
684 void dump(raw_fd_ostream &OS) const;
685
687 FuncName = Name;
688 FuncHash = Hash;
689 }
690
691 Error accumulateCounts(const std::string &BaseFilename,
692 const std::string &TestFilename, bool IsCS);
693 void addOneMismatch(const CountSumOrPercent &MismatchFunc);
694 void addOneUnique(const CountSumOrPercent &UniqueFunc);
695
696 static inline double score(uint64_t Val1, uint64_t Val2, double Sum1,
697 double Sum2) {
698 if (Sum1 < 1.0f || Sum2 < 1.0f)
699 return 0.0f;
700 return std::min(Val1 / Sum1, Val2 / Sum2);
701 }
702};
703
704// This is used to filter the functions whose overlap information
705// to be output.
708 const std::string NameFilter;
709};
710
712 /// Value profiling data pairs at a given value site.
713 std::list<InstrProfValueData> ValueData;
714
716 template <class InputIterator>
717 InstrProfValueSiteRecord(InputIterator F, InputIterator L)
718 : ValueData(F, L) {}
719
720 /// Sort ValueData ascending by Value
722 ValueData.sort(
723 [](const InstrProfValueData &left, const InstrProfValueData &right) {
724 return left.Value < right.Value;
725 });
726 }
727 /// Sort ValueData Descending by Count
728 inline void sortByCount();
729
730 /// Merge data from another InstrProfValueSiteRecord
731 /// Optionally scale merged counts by \p Weight.
732 void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
733 function_ref<void(instrprof_error)> Warn);
734 /// Scale up value profile data counts by N (Numerator) / D (Denominator).
736
737 /// Compute the overlap b/w this record and Input record.
738 void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
739 OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);
740};
741
742/// Profiling information for a single function.
744 std::vector<uint64_t> Counts;
745
746 InstrProfRecord() = default;
747 InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
750 : Counts(RHS.Counts),
751 ValueData(RHS.ValueData
752 ? std::make_unique<ValueProfData>(*RHS.ValueData)
753 : nullptr) {}
756 Counts = RHS.Counts;
757 if (!RHS.ValueData) {
758 ValueData = nullptr;
759 return *this;
760 }
761 if (!ValueData)
762 ValueData = std::make_unique<ValueProfData>(*RHS.ValueData);
763 else
764 *ValueData = *RHS.ValueData;
765 return *this;
766 }
767
768 /// Return the number of value profile kinds with non-zero number
769 /// of profile sites.
770 inline uint32_t getNumValueKinds() const;
771 /// Return the number of instrumented sites for ValueKind.
772 inline uint32_t getNumValueSites(uint32_t ValueKind) const;
773
774 /// Return the total number of ValueData for ValueKind.
775 inline uint32_t getNumValueData(uint32_t ValueKind) const;
776
777 /// Return the number of value data collected for ValueKind at profiling
778 /// site: Site.
780 uint32_t Site) const;
781
782 /// Return the array of profiled values at \p Site. If \p TotalC
783 /// is not null, the total count of all target values at this site
784 /// will be stored in \c *TotalC.
785 inline std::unique_ptr<InstrProfValueData[]>
786 getValueForSite(uint32_t ValueKind, uint32_t Site,
787 uint64_t *TotalC = nullptr) const;
788
789 /// Get the target value/counts of kind \p ValueKind collected at site
790 /// \p Site and store the result in array \p Dest. Return the total
791 /// counts of all target values at this site.
792 inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,
793 uint32_t Site) const;
794
795 /// Reserve space for NumValueSites sites.
796 inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
797
798 /// Add ValueData for ValueKind at value Site.
799 void addValueData(uint32_t ValueKind, uint32_t Site,
800 InstrProfValueData *VData, uint32_t N,
801 InstrProfSymtab *SymTab);
802
803 /// Merge the counts in \p Other into this one.
804 /// Optionally scale merged counts by \p Weight.
805 void merge(InstrProfRecord &Other, uint64_t Weight,
806 function_ref<void(instrprof_error)> Warn);
807
808 /// Scale up profile counts (including value profile data) by
809 /// a factor of (N / D).
811
812 /// Sort value profile data (per site) by count.
814 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
815 for (auto &SR : getValueSitesForKind(Kind))
816 SR.sortByCount();
817 }
818
819 /// Clear value data entries and edge counters.
820 void Clear() {
821 Counts.clear();
823 }
824
825 /// Clear value data entries
826 void clearValueData() { ValueData = nullptr; }
827
828 /// Compute the sums of all counts and store in Sum.
829 void accumulateCounts(CountSumOrPercent &Sum) const;
830
831 /// Compute the overlap b/w this IntrprofRecord and Other.
833 OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);
834
835 /// Compute the overlap of value profile counts.
836 void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
837 OverlapStats &Overlap,
838 OverlapStats &FuncLevelOverlap);
839
844 };
848 };
850 uint64_t FirstCount = Counts[0];
851 if (FirstCount == (uint64_t)HotFunctionVal)
852 return PseudoHot;
853 if (FirstCount == (uint64_t)WarmFunctionVal)
854 return PseudoWarm;
855 return NotPseudo;
856 }
858 if (Kind == PseudoHot)
860 else if (Kind == PseudoWarm)
862 }
863
864private:
865 struct ValueProfData {
866 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
867 std::vector<InstrProfValueSiteRecord> MemOPSizes;
868 };
869 std::unique_ptr<ValueProfData> ValueData;
870
871 MutableArrayRef<InstrProfValueSiteRecord>
872 getValueSitesForKind(uint32_t ValueKind) {
873 // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
874 // implemented in LLVM) to call the const overload of this function, then
875 // cast away the constness from the result.
876 auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
877 ValueKind);
878 return MutableArrayRef(
879 const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
880 }
881 ArrayRef<InstrProfValueSiteRecord>
882 getValueSitesForKind(uint32_t ValueKind) const {
883 if (!ValueData)
884 return std::nullopt;
885 switch (ValueKind) {
886 case IPVK_IndirectCallTarget:
887 return ValueData->IndirectCallSites;
888 case IPVK_MemOPSize:
889 return ValueData->MemOPSizes;
890 default:
891 llvm_unreachable("Unknown value kind!");
892 }
893 }
894
895 std::vector<InstrProfValueSiteRecord> &
896 getOrCreateValueSitesForKind(uint32_t ValueKind) {
897 if (!ValueData)
898 ValueData = std::make_unique<ValueProfData>();
899 switch (ValueKind) {
900 case IPVK_IndirectCallTarget:
901 return ValueData->IndirectCallSites;
902 case IPVK_MemOPSize:
903 return ValueData->MemOPSizes;
904 default:
905 llvm_unreachable("Unknown value kind!");
906 }
907 }
908
909 // Map indirect call target name hash to name string.
910 uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
911 InstrProfSymtab *SymTab);
912
913 // Merge Value Profile data from Src record to this record for ValueKind.
914 // Scale merged value counts by \p Weight.
915 void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
916 uint64_t Weight,
917 function_ref<void(instrprof_error)> Warn);
918
919 // Scale up value profile data count by N (Numerator) / D (Denominator).
920 void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
921 function_ref<void(instrprof_error)> Warn);
922};
923
927
928 // We reserve this bit as the flag for context sensitive profile record.
929 static const int CS_FLAG_IN_FUNC_HASH = 60;
930
933 std::vector<uint64_t> Counts)
935
936 static bool hasCSFlagInHash(uint64_t FuncHash) {
937 return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1);
938 }
939 static void setCSFlagInHash(uint64_t &FuncHash) {
940 FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH);
941 }
942};
943
945 uint32_t NumValueKinds = 0;
946 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
947 NumValueKinds += !(getValueSitesForKind(Kind).empty());
948 return NumValueKinds;
949}
950
952 uint32_t N = 0;
953 for (const auto &SR : getValueSitesForKind(ValueKind))
954 N += SR.ValueData.size();
955 return N;
956}
957
959 return getValueSitesForKind(ValueKind).size();
960}
961
963 uint32_t Site) const {
964 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
965}
966
967std::unique_ptr<InstrProfValueData[]>
969 uint64_t *TotalC) const {
970 uint64_t Dummy = 0;
971 uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC);
972 uint32_t N = getNumValueDataForSite(ValueKind, Site);
973 if (N == 0) {
974 TotalCount = 0;
975 return std::unique_ptr<InstrProfValueData[]>(nullptr);
976 }
977
978 auto VD = std::make_unique<InstrProfValueData[]>(N);
979 TotalCount = getValueForSite(VD.get(), ValueKind, Site);
980
981 return VD;
982}
983
985 uint32_t ValueKind,
986 uint32_t Site) const {
987 uint32_t I = 0;
988 uint64_t TotalCount = 0;
989 for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
990 Dest[I].Value = V.Value;
991 Dest[I].Count = V.Count;
992 TotalCount = SaturatingAdd(TotalCount, V.Count);
993 I++;
994 }
995 return TotalCount;
996}
997
998void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
999 if (!NumValueSites)
1000 return;
1001 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
1002}
1003
1006}
1007
1008// Include definitions for value profile data
1009#define INSTR_PROF_VALUE_PROF_DATA
1011
1013 ValueData.sort(
1014 [](const InstrProfValueData &left, const InstrProfValueData &right) {
1015 return left.Count > right.Count;
1016 });
1017 // Now truncate
1018 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
1019 if (ValueData.size() > max_s)
1020 ValueData.resize(max_s);
1021}
1022
1023namespace IndexedInstrProf {
1024
1025enum class HashT : uint32_t {
1026 MD5,
1027 Last = MD5
1028};
1029
1031 switch (Type) {
1032 case HashT::MD5:
1033 return MD5Hash(K);
1034 }
1035 llvm_unreachable("Unhandled hash type");
1036}
1037
1038const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
1039
1041 // Version 1 is the first version. In this version, the value of
1042 // a key/value pair can only include profile data of a single function.
1043 // Due to this restriction, the number of block counters for a given
1044 // function is not recorded but derived from the length of the value.
1046 // The version 2 format supports recording profile data of multiple
1047 // functions which share the same key in one value field. To support this,
1048 // the number block counters is recorded as an uint64_t field right after the
1049 // function structural hash.
1051 // Version 3 supports value profile data. The value profile data is expected
1052 // to follow the block counter profile data.
1054 // In this version, profile summary data \c IndexedInstrProf::Summary is
1055 // stored after the profile header.
1057 // In this version, the frontend PGO stable hash algorithm defaults to V2.
1059 // In this version, the frontend PGO stable hash algorithm got fixed and
1060 // may produce hashes different from Version5.
1062 // An additional counter is added around logical operators.
1064 // An additional (optional) memory profile type is added.
1066 // Binary ids are added.
1068 // An additional (optional) temporal profile traces section is added.
1070 // The current version is 10.
1071 CurrentVersion = INSTR_PROF_INDEX_VERSION
1074
1076
1078
1079// This structure defines the file header of the LLVM profile
1080// data file in indexed-format.
1081struct Header {
1084 uint64_t Unused; // Becomes unused since version 4
1090 // New fields should only be added at the end to ensure that the size
1091 // computation is correct. The methods below need to be updated to ensure that
1092 // the new field is read correctly.
1093
1094 // Reads a header struct from the buffer.
1095 static Expected<Header> readFromBuffer(const unsigned char *Buffer);
1096
1097 // Returns the size of the header in bytes for all valid fields based on the
1098 // version. I.e a older version header will return a smaller size.
1099 size_t size() const;
1100
1101 // Returns the format version in little endian. The header retains the version
1102 // in native endian of the compiler runtime.
1103 uint64_t formatVersion() const;
1104};
1105
1106// Profile summary data recorded in the profile data file in indexed
1107// format. It is introduced in version 4. The summary data follows
1108// right after the profile file header.
1109struct Summary {
1110 struct Entry {
1111 uint64_t Cutoff; ///< The required percentile of total execution count.
1112 uint64_t
1113 MinBlockCount; ///< The minimum execution count for this percentile.
1114 uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
1115 };
1116 // The field kind enumerator to assigned value mapping should remain
1117 // unchanged when a new kind is added or an old kind gets deleted in
1118 // the future.
1120 /// The total number of functions instrumented.
1122 /// Total number of instrumented blocks/edges.
1124 /// The maximal execution count among all functions.
1125 /// This field does not exist for profile data from IR based
1126 /// instrumentation.
1128 /// Max block count of the program.
1130 /// Max internal block count of the program (excluding entry blocks).
1132 /// The sum of all instrumented block counts.
1136
1137 // The number of summmary fields following the summary header.
1139 // The number of Cutoff Entries (Summary::Entry) following summary fields.
1141
1142 Summary() = delete;
1143 Summary(uint32_t Size) { memset(this, 0, Size); }
1144
1145 void operator delete(void *ptr) { ::operator delete(ptr); }
1146
1148 return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
1149 NumSumFields * sizeof(uint64_t);
1150 }
1151
1153 return reinterpret_cast<const uint64_t *>(this + 1);
1154 }
1155
1157 return reinterpret_cast<uint64_t *>(this + 1);
1158 }
1159
1160 const Entry *getCutoffEntryBase() const {
1161 return reinterpret_cast<const Entry *>(
1163 }
1164
1166 return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
1167 }
1168
1170 return getSummaryDataBase()[K];
1171 }
1172
1174 getSummaryDataBase()[K] = V;
1175 }
1176
1177 const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
1178
1180 Entry &ER = getCutoffEntryBase()[I];
1181 ER.Cutoff = E.Cutoff;
1182 ER.MinBlockCount = E.MinCount;
1183 ER.NumBlocks = E.NumCounts;
1184 }
1185};
1186
1187inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
1188 return std::unique_ptr<Summary>(new (::operator new(TotalSize))
1189 Summary(TotalSize));
1190}
1191
1192} // end namespace IndexedInstrProf
1193
1194namespace RawInstrProf {
1195
1196// Version 1: First version
1197// Version 2: Added value profile data section. Per-function control data
1198// struct has more fields to describe value profile information.
1199// Version 3: Compressed name section support. Function PGO name reference
1200// from control data struct is changed from raw pointer to Name's MD5 value.
1201// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
1202// raw header.
1203// Version 5: Bit 60 of FuncHash is reserved for the flag for the context
1204// sensitive records.
1205// Version 6: Added binary id.
1206// Version 7: Reorder binary id and include version in signature.
1207// Version 8: Use relative counter pointer.
1208const uint64_t Version = INSTR_PROF_RAW_VERSION;
1209
1210template <class IntPtrT> inline uint64_t getMagic();
1211template <> inline uint64_t getMagic<uint64_t>() {
1212 return INSTR_PROF_RAW_MAGIC_64;
1213}
1214
1215template <> inline uint64_t getMagic<uint32_t>() {
1216 return INSTR_PROF_RAW_MAGIC_32;
1217}
1218
1219// Per-function profile data header/control structure.
1220// The definition should match the structure defined in
1221// compiler-rt/lib/profile/InstrProfiling.h.
1222// It should also match the synthesized type in
1223// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
1224template <class IntPtrT> struct alignas(8) ProfileData {
1225 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1227};
1228
1229// File header structure of the LLVM profile data in raw format.
1230// The definition should match the header referenced in
1231// compiler-rt/lib/profile/InstrProfilingFile.c and
1232// InstrProfilingBuffer.c.
1233struct Header {
1234#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1236};
1237
1238} // end namespace RawInstrProf
1239
1240// Parse MemOP Size range option.
1241void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
1242 int64_t &RangeLast);
1243
1244// Create the variable for the profile file name.
1245void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
1246
1247// Whether to compress function names in profile records, and filenames in
1248// code coverage mappings. Used by the Instrumentation library and unit tests.
1250
1251} // end namespace llvm
1252#endif // LLVM_PROFILEDATA_INSTRPROF_H
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
uint64_t Addr
std::string Name
uint64_t Size
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Machine Check Debug Module
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
StringSet - A set-like wrapper for the StringMap.
Value * RHS
Value * LHS
Base class for user error types.
Definition: Error.h:348
Lightweight error class with error context and mandatory checking.
Definition: Error.h:156
static ErrorSuccess success()
Create a success value.
Definition: Error.h:330
Tagged union holding either a T or a Error.
Definition: Error.h:470
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: InstrProf.h:357
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
Definition: InstrProf.h:369
const std::string & getMessage() const
Definition: InstrProf.h:364
static char ID
Definition: InstrProf.h:380
instrprof_error get() const
Definition: InstrProf.h:363
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: InstrProf.h:359
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:260
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
Definition: InstrProf.h:350
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:459
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:557
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:428
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
Definition: InstrProf.h:461
StringRef getOrigFuncName(uint64_t FuncMD5Hash)
Return the function's original assembly name by stripping off the prefix attached (to symbols with pr...
Definition: InstrProf.h:639
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Definition: InstrProf.h:535
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Definition: InstrProf.h:573
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
Definition: InstrProf.h:627
StringRef getNameData() const
Return the name section data.
Definition: InstrProf.h:570
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:520
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
Definition: InstrProf.h:610
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
Definition: MD5.h:41
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Error takeError()
Return the first encountered error and reset FirstError to a success value.
Definition: InstrProf.h:433
unsigned getNumHashMismatches() const
Get the number of hash mismatches.
Definition: InstrProf.h:418
unsigned getNumCountMismatches() const
Get the number of count mismatches.
Definition: InstrProf.h:421
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
Definition: InstrProf.cpp:235
unsigned getNumValueSiteCountMismatches() const
Get the number of value site count mismatches.
Definition: InstrProf.h:427
unsigned getNumCounterOverflows() const
Get the number of counter overflows.
Definition: InstrProf.h:424
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
Definition: StringMap.h:212
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition: StringRef.h:607
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:375
static constexpr size_t npos
Definition: StringRef.h:52
StringSet - A wrapper for StringMap that provides set-like functionality.
Definition: StringSet.h:23
std::pair< typename Base::iterator, bool > insert(StringRef key)
Definition: StringSet.h:34
ObjectFormatType
Definition: Triple.h:280
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
An efficient, type-erasing, non-owning reference to a callable.
This is a value type class that represents a single section in the list of sections in the object fil...
Definition: ObjectFile.h:80
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1187
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1077
const uint64_t Version
Definition: InstrProf.h:1073
const uint64_t Magic
Definition: InstrProf.h:1038
const HashT HashType
Definition: InstrProf.h:1075
uint64_t getMagic()
const uint64_t Version
Definition: InstrProf.h:1208
uint64_t getMagic< uint32_t >()
Definition: InstrProf.h:1215
uint64_t getMagic< uint64_t >()
Definition: InstrProf.h:1211
constexpr size_t NameSize
Definition: XCOFF.h:29
static const bool IsLittleEndianHost
Definition: SwapByteOrder.h:70
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1066
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:90
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
Definition: InstrProf.cpp:497
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Definition: InstrProf.cpp:304
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:154
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1126
std::error_code make_error_code(BitcodeError E)
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:272
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
Definition: InstrProf.h:113
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
Definition: Error.h:966
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:327
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1122
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:93
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:121
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:216
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition: InstrProf.h:64
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:149
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:99
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:164
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:383
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1021
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:160
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:130
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1744
InstrProfSectKind
Definition: InstrProf.h:58
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:96
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:480
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:169
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:85
support::endianness getHostEndianness()
Definition: InstrProf.h:1004
instrprof_error
Definition: InstrProf.h:310
InstrProfValueKind
Definition: InstrProf.h:244
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:141
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:2038
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:183
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1138
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1186
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
Definition: MD5.h:109
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1209
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1946
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:442
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:79
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:136
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:338
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:583
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:106
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1164
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast)
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:102
InstrProfKind
An enum describing the attributes of an instrumented profile.
Definition: InstrProf.h:287
Definition: BitVector.h:858
#define N
double ValueCounts[IPVK_Last - IPVK_First+1]
Definition: InstrProf.h:652
uint64_t formatVersion() const
Definition: InstrProf.cpp:1354
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1359
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1111
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1114
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1113
const Entry * getCutoffEntryBase() const
Definition: InstrProf.h:1160
uint64_t get(SummaryFieldKind K) const
Definition: InstrProf.h:1169
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:1173
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:1179
@ TotalNumFunctions
The total number of functions instrumented.
Definition: InstrProf.h:1121
@ TotalNumBlocks
Total number of instrumented blocks/edges.
Definition: InstrProf.h:1123
@ MaxFunctionCount
The maximal execution count among all functions.
Definition: InstrProf.h:1127
@ TotalBlockCount
The sum of all instrumented block counts.
Definition: InstrProf.h:1133
@ MaxBlockCount
Max block count of the program.
Definition: InstrProf.h:1129
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
Definition: InstrProf.h:1131
const uint64_t * getSummaryDataBase() const
Definition: InstrProf.h:1152
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1147
const Entry & getEntry(uint32_t I) const
Definition: InstrProf.h:1177
Profiling information for a single function.
Definition: InstrProf.h:743
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:590
std::vector< uint64_t > Counts
Definition: InstrProf.h:744
CountPseudoKind getCountPseudoKind() const
Definition: InstrProf.h:849
InstrProfRecord(std::vector< uint64_t > Counts)
Definition: InstrProf.h:747
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:539
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:958
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
Definition: InstrProf.h:944
void setPseudoCount(CountPseudoKind Kind)
Definition: InstrProf.h:857
InstrProfRecord(InstrProfRecord &&)=default
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
Definition: InstrProf.h:951
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:712
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:792
InstrProfRecord & operator=(const InstrProfRecord &RHS)
Definition: InstrProf.h:755
void clearValueData()
Clear value data entries.
Definition: InstrProf.h:826
InstrProfRecord(const InstrProfRecord &RHS)
Definition: InstrProf.h:749
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:962
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
Definition: InstrProf.h:998
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:608
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:813
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
Definition: InstrProf.h:968
InstrProfRecord & operator=(InstrProfRecord &&)=default
void Clear()
Clear value data entries and edge counters.
Definition: InstrProf.h:820
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:763
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
Definition: InstrProf.h:717
void sortByCount()
Sort ValueData Descending by Count.
Definition: InstrProf.h:1012
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:721
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:659
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:559
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:713
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:681
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:936
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
Definition: InstrProf.h:932
static void setCSFlagInHash(uint64_t &FuncHash)
Definition: InstrProf.h:939
static const int CS_FLAG_IN_FUNC_HASH
Definition: InstrProf.h:929
const std::string NameFilter
Definition: InstrProf.h:708
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1254
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:696
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1226
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1273
CountSumOrPercent Overlap
Definition: InstrProf.h:670
CountSumOrPercent Base
Definition: InstrProf.h:666
uint64_t FuncHash
Definition: InstrProf.h:677
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1264
const std::string * BaseFilename
Definition: InstrProf.h:674
const std::string * TestFilename
Definition: InstrProf.h:675
void setFuncInfo(StringRef Name, uint64_t Hash)
Definition: InstrProf.h:686
CountSumOrPercent Unique
Definition: InstrProf.h:672
CountSumOrPercent Mismatch
Definition: InstrProf.h:671
StringRef FuncName
Definition: InstrProf.h:676
OverlapStatsLevel Level
Definition: InstrProf.h:673
OverlapStats(OverlapStatsLevel L=ProgramLevel)
Definition: InstrProf.h:680
CountSumOrPercent Test
Definition: InstrProf.h:668
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:339
std::vector< uint64_t > FunctionNameRefs
Definition: InstrProf.h:341
Function object to check whether the first component of a container supported by std::get (like std::...
Definition: STLExtras.h:1537