LCOV - code coverage report
Current view: top level - include/llvm/ProfileData - InstrProf.h (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 175 187 93.6 %
Date: 2017-09-14 15:23:50 Functions: 25 28 89.3 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // Instrumentation-based profiling data is generated by instrumented
      11             : // binaries through library functions in compiler-rt, and read by the clang
      12             : // frontend to feed PGO.
      13             : //
      14             : //===----------------------------------------------------------------------===//
      15             : 
      16             : #ifndef LLVM_PROFILEDATA_INSTRPROF_H
      17             : #define LLVM_PROFILEDATA_INSTRPROF_H
      18             : 
      19             : #include "llvm/ADT/ArrayRef.h"
      20             : #include "llvm/ADT/STLExtras.h"
      21             : #include "llvm/ADT/StringRef.h"
      22             : #include "llvm/ADT/StringSet.h"
      23             : #include "llvm/ADT/Triple.h"
      24             : #include "llvm/IR/GlobalValue.h"
      25             : #include "llvm/IR/ProfileSummary.h"
      26             : #include "llvm/ProfileData/InstrProfData.inc"
      27             : #include "llvm/Support/Compiler.h"
      28             : #include "llvm/Support/Endian.h"
      29             : #include "llvm/Support/Error.h"
      30             : #include "llvm/Support/ErrorHandling.h"
      31             : #include "llvm/Support/Host.h"
      32             : #include "llvm/Support/MD5.h"
      33             : #include "llvm/Support/MathExtras.h"
      34             : #include "llvm/Support/raw_ostream.h"
      35             : #include <algorithm>
      36             : #include <cassert>
      37             : #include <cstddef>
      38             : #include <cstdint>
      39             : #include <cstring>
      40             : #include <list>
      41             : #include <memory>
      42             : #include <string>
      43             : #include <system_error>
      44             : #include <utility>
      45             : #include <vector>
      46             : 
      47             : namespace llvm {
      48             : 
      49             : class Function;
      50             : class GlobalVariable;
      51             : struct InstrProfRecord;
      52             : class InstrProfSymtab;
      53             : class Instruction;
      54             : class MDNode;
      55             : class Module;
      56             : 
      57             : enum InstrProfSectKind {
      58             : #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
      59             : #include "llvm/ProfileData/InstrProfData.inc"
      60             : };
      61             : 
      62             : /// Return the name of the profile section corresponding to \p IPSK.
      63             : ///
      64             : /// The name of the section depends on the object format type \p OF. If
      65             : /// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
      66             : /// be added to the section name (this is the default).
      67             : std::string getInstrProfSectionName(InstrProfSectKind IPSK,
      68             :                                     Triple::ObjectFormatType OF,
      69             :                                     bool AddSegmentInfo = true);
      70             : 
      71             : /// Return the name profile runtime entry point to do value profiling
      72             : /// for a given site.
      73             : inline StringRef getInstrProfValueProfFuncName() {
      74      223733 :   return INSTR_PROF_VALUE_PROF_FUNC_STR;
      75             : }
      76             : 
      77             : /// Return the name profile runtime entry point to do value range profiling.
      78             : inline StringRef getInstrProfValueRangeProfFuncName() {
      79           2 :   return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
      80             : }
      81             : 
      82             : /// Return the name prefix of variables containing instrumented function names.
      83        1091 : inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
      84             : 
      85             : /// Return the name prefix of variables containing per-function control data.
      86         335 : inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
      87             : 
      88             : /// Return the name prefix of profile counter variables.
      89         340 : inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
      90             : 
      91             : /// Return the name prefix of value profile variables.
      92          16 : inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
      93             : 
      94             : /// Return the name of value profile node array variables:
      95          15 : inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
      96             : 
      97             : /// Return the name prefix of the COMDAT group for instrumentation variables
      98             : /// associated with a COMDAT function.
      99          52 : inline StringRef getInstrProfComdatPrefix() { return "__profv_"; }
     100             : 
     101             : /// Return the name of the variable holding the strings (possibly compressed)
     102             : /// of all function's PGO names.
     103             : inline StringRef getInstrProfNamesVarName() {
     104         138 :   return "__llvm_prf_nm";
     105             : }
     106             : 
     107             : /// Return the name of a covarage mapping variable (internal linkage)
     108             : /// for each instrumented source module. Such variables are allocated
     109             : /// in the __llvm_covmap section.
     110             : inline StringRef getCoverageMappingVarName() {
     111          53 :   return "__llvm_coverage_mapping";
     112             : }
     113             : 
     114             : /// Return the name of the internal variable recording the array
     115             : /// of PGO name vars referenced by the coverage mapping. The owning
     116             : /// functions of those names are not emitted by FE (e.g, unused inline
     117             : /// functions.)
     118             : inline StringRef getCoverageUnusedNamesVarName() {
     119         148 :   return "__llvm_coverage_names";
     120             : }
     121             : 
     122             : /// Return the name of function that registers all the per-function control
     123             : /// data at program startup time by calling __llvm_register_function. This
     124             : /// function has internal linkage and is called by  __llvm_profile_init
     125             : /// runtime method. This function is not generated for these platforms:
     126             : /// Darwin, Linux, and FreeBSD.
     127             : inline StringRef getInstrProfRegFuncsName() {
     128         160 :   return "__llvm_profile_register_functions";
     129             : }
     130             : 
     131             : /// Return the name of the runtime interface that registers per-function control
     132             : /// data for one instrumented function.
     133             : inline StringRef getInstrProfRegFuncName() {
     134          22 :   return "__llvm_profile_register_function";
     135             : }
     136             : 
     137             : /// Return the name of the runtime interface that registers the PGO name strings.
     138             : inline StringRef getInstrProfNamesRegFuncName() {
     139          22 :   return "__llvm_profile_register_names_function";
     140             : }
     141             : 
     142             : /// Return the name of the runtime initialization method that is generated by
     143             : /// the compiler. The function calls __llvm_profile_register_functions and
     144             : /// __llvm_profile_override_default_filename functions if needed. This function
     145             : /// has internal linkage and invoked at startup time via init_array.
     146          22 : inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
     147             : 
     148             : /// Return the name of the hook variable defined in profile runtime library.
     149             : /// A reference to the variable causes the linker to link in the runtime
     150             : /// initialization module (which defines the hook variable).
     151             : inline StringRef getInstrProfRuntimeHookVarName() {
     152         111 :   return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
     153             : }
     154             : 
     155             : /// Return the name of the compiler generated function that references the
     156             : /// runtime hook variable. The function is a weak global.
     157             : inline StringRef getInstrProfRuntimeHookVarUseFuncName() {
     158          52 :   return "__llvm_profile_runtime_user";
     159             : }
     160             : 
     161             : /// Return the marker used to separate PGO names during serialization.
     162         213 : inline StringRef getInstrProfNameSeparator() { return "\01"; }
     163             : 
     164             : /// Return the modified name for function \c F suitable to be
     165             : /// used the key for profile lookup. Variable \c InLTO indicates if this
     166             : /// is called in LTO optimization passes.
     167             : std::string getPGOFuncName(const Function &F, bool InLTO = false,
     168             :                            uint64_t Version = INSTR_PROF_INDEX_VERSION);
     169             : 
     170             : /// Return the modified name for a function suitable to be
     171             : /// used the key for profile lookup. The function's original
     172             : /// name is \c RawFuncName and has linkage of type \c Linkage.
     173             : /// The function is defined in module \c FileName.
     174             : std::string getPGOFuncName(StringRef RawFuncName,
     175             :                            GlobalValue::LinkageTypes Linkage,
     176             :                            StringRef FileName,
     177             :                            uint64_t Version = INSTR_PROF_INDEX_VERSION);
     178             : 
     179             : /// Return the name of the global variable used to store a function
     180             : /// name in PGO instrumentation. \c FuncName is the name of the function
     181             : /// returned by the \c getPGOFuncName call.
     182             : std::string getPGOFuncNameVarName(StringRef FuncName,
     183             :                                   GlobalValue::LinkageTypes Linkage);
     184             : 
     185             : /// Create and return the global variable for function name used in PGO
     186             : /// instrumentation. \c FuncName is the name of the function returned
     187             : /// by \c getPGOFuncName call.
     188             : GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
     189             : 
     190             : /// Create and return the global variable for function name used in PGO
     191             : /// instrumentation.  /// \c FuncName is the name of the function
     192             : /// returned by \c getPGOFuncName call, \c M is the owning module,
     193             : /// and \c Linkage is the linkage of the instrumented function.
     194             : GlobalVariable *createPGOFuncNameVar(Module &M,
     195             :                                      GlobalValue::LinkageTypes Linkage,
     196             :                                      StringRef PGOFuncName);
     197             : 
     198             : /// Return the initializer in string of the PGO name var \c NameVar.
     199             : StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
     200             : 
     201             : /// Given a PGO function name, remove the filename prefix and return
     202             : /// the original (static) function name.
     203             : StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
     204             :                                    StringRef FileName = "<unknown>");
     205             : 
     206             : /// Given a vector of strings (function PGO names) \c NameStrs, the
     207             : /// method generates a combined string \c Result thatis ready to be
     208             : /// serialized.  The \c Result string is comprised of three fields:
     209             : /// The first field is the legnth of the uncompressed strings, and the
     210             : /// the second field is the length of the zlib-compressed string.
     211             : /// Both fields are encoded in ULEB128.  If \c doCompress is false, the
     212             : ///  third field is the uncompressed strings; otherwise it is the
     213             : /// compressed string. When the string compression is off, the
     214             : /// second field will have value zero.
     215             : Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
     216             :                                 bool doCompression, std::string &Result);
     217             : 
     218             : /// Produce \c Result string with the same format described above. The input
     219             : /// is vector of PGO function name variables that are referenced.
     220             : Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
     221             :                                 std::string &Result, bool doCompression = true);
     222             : 
     223             : /// \c NameStrings is a string composed of one of more sub-strings encoded in
     224             : /// the format described above. The substrings are separated by 0 or more zero
     225             : /// bytes. This method decodes the string and populates the \c Symtab.
     226             : Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
     227             : 
     228             : /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
     229             : /// set in IR PGO compilation.
     230             : bool isIRPGOFlagSet(const Module *M);
     231             : 
     232             : /// Check if we can safely rename this Comdat function. Instances of the same
     233             : /// comdat function may have different control flows thus can not share the
     234             : /// same counter variable.
     235             : bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
     236             : 
     237             : enum InstrProfValueKind : uint32_t {
     238             : #define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value,
     239             : #include "llvm/ProfileData/InstrProfData.inc"
     240             : };
     241             : 
     242             : /// Get the value profile data for value site \p SiteIdx from \p InstrProfR
     243             : /// and annotate the instruction \p Inst with the value profile meta data.
     244             : /// Annotate up to \p MaxMDCount (default 3) number of records per value site.
     245             : void annotateValueSite(Module &M, Instruction &Inst,
     246             :                        const InstrProfRecord &InstrProfR,
     247             :                        InstrProfValueKind ValueKind, uint32_t SiteIndx,
     248             :                        uint32_t MaxMDCount = 3);
     249             : 
     250             : /// Same as the above interface but using an ArrayRef, as well as \p Sum.
     251             : void annotateValueSite(Module &M, Instruction &Inst,
     252             :                        ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
     253             :                        InstrProfValueKind ValueKind, uint32_t MaxMDCount);
     254             : 
     255             : /// Extract the value profile data from \p Inst which is annotated with
     256             : /// value profile meta data. Return false if there is no value data annotated,
     257             : /// otherwise  return true.
     258             : bool getValueProfDataFromInst(const Instruction &Inst,
     259             :                               InstrProfValueKind ValueKind,
     260             :                               uint32_t MaxNumValueData,
     261             :                               InstrProfValueData ValueData[],
     262             :                               uint32_t &ActualNumValueData, uint64_t &TotalC);
     263             : 
     264         974 : inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
     265             : 
     266             : /// Return the PGOFuncName meta data associated with a function.
     267             : MDNode *getPGOFuncNameMetadata(const Function &F);
     268             : 
     269             : /// Create the PGOFuncName meta data if PGOFuncName is different from
     270             : /// function's raw name. This should only apply to internal linkage functions
     271             : /// declared by users only.
     272             : void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
     273             : 
     274             : /// Check if we can use Comdat for profile variables. This will eliminate
     275             : /// the duplicated profile variables for Comdat functions.
     276             : bool needsComdatForCounter(const Function &F, const Module &M);
     277             : 
     278             : const std::error_category &instrprof_category();
     279             : 
     280             : enum class instrprof_error {
     281             :   success = 0,
     282             :   eof,
     283             :   unrecognized_format,
     284             :   bad_magic,
     285             :   bad_header,
     286             :   unsupported_version,
     287             :   unsupported_hash_type,
     288             :   too_large,
     289             :   truncated,
     290             :   malformed,
     291             :   unknown_function,
     292             :   hash_mismatch,
     293             :   count_mismatch,
     294             :   counter_overflow,
     295             :   value_site_count_mismatch,
     296             :   compress_failed,
     297             :   uncompress_failed,
     298             :   empty_raw_profile,
     299             :   zlib_unavailable
     300             : };
     301             : 
     302             : inline std::error_code make_error_code(instrprof_error E) {
     303           0 :   return std::error_code(static_cast<int>(E), instrprof_category());
     304             : }
     305             : 
     306         702 : class InstrProfError : public ErrorInfo<InstrProfError> {
     307             : public:
     308         724 :   InstrProfError(instrprof_error Err) : Err(Err) {
     309             :     assert(Err != instrprof_error::success && "Not an error");
     310             :   }
     311             : 
     312             :   std::string message() const override;
     313             : 
     314           0 :   void log(raw_ostream &OS) const override { OS << message(); }
     315             : 
     316           0 :   std::error_code convertToErrorCode() const override {
     317           0 :     return make_error_code(Err);
     318             :   }
     319             : 
     320             :   instrprof_error get() const { return Err; }
     321             : 
     322             :   /// Consume an Error and return the raw enum value contained within it. The
     323             :   /// Error must either be a success value, or contain a single InstrProfError.
     324         329 :   static instrprof_error take(Error E) {
     325         329 :     auto Err = instrprof_error::success;
     326        1316 :     handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) {
     327             :       assert(Err == instrprof_error::success && "Multiple errors encountered");
     328         329 :       Err = IPE.get();
     329             :     });
     330         329 :     return Err;
     331             :   }
     332             : 
     333             :   static char ID;
     334             : 
     335             : private:
     336             :   instrprof_error Err;
     337             : };
     338             : 
     339             : class SoftInstrProfErrors {
     340             :   /// Count the number of soft instrprof_errors encountered and keep track of
     341             :   /// the first such error for reporting purposes.
     342             : 
     343             :   /// The first soft error encountered.
     344             :   instrprof_error FirstError = instrprof_error::success;
     345             : 
     346             :   /// The number of hash mismatches.
     347             :   unsigned NumHashMismatches = 0;
     348             : 
     349             :   /// The number of count mismatches.
     350             :   unsigned NumCountMismatches = 0;
     351             : 
     352             :   /// The number of counter overflows.
     353             :   unsigned NumCounterOverflows = 0;
     354             : 
     355             :   /// The number of value site count mismatches.
     356             :   unsigned NumValueSiteCountMismatches = 0;
     357             : 
     358             : public:
     359             :   SoftInstrProfErrors() = default;
     360             : 
     361             :   ~SoftInstrProfErrors() {
     362             :     assert(FirstError == instrprof_error::success &&
     363             :            "Unchecked soft error encountered");
     364             :   }
     365             : 
     366             :   /// Track a soft error (\p IE) and increment its associated counter.
     367             :   void addError(instrprof_error IE);
     368             : 
     369             :   /// Get the number of hash mismatches.
     370             :   unsigned getNumHashMismatches() const { return NumHashMismatches; }
     371             : 
     372             :   /// Get the number of count mismatches.
     373             :   unsigned getNumCountMismatches() const { return NumCountMismatches; }
     374             : 
     375             :   /// Get the number of counter overflows.
     376             :   unsigned getNumCounterOverflows() const { return NumCounterOverflows; }
     377             : 
     378             :   /// Get the number of value site count mismatches.
     379             :   unsigned getNumValueSiteCountMismatches() const {
     380             :     return NumValueSiteCountMismatches;
     381             :   }
     382             : 
     383             :   /// Return the first encountered error and reset FirstError to a success
     384             :   /// value.
     385             :   Error takeError() {
     386             :     if (FirstError == instrprof_error::success)
     387             :       return Error::success();
     388             :     auto E = make_error<InstrProfError>(FirstError);
     389             :     FirstError = instrprof_error::success;
     390             :     return E;
     391             :   }
     392             : };
     393             : 
     394             : namespace object {
     395             : 
     396             : class SectionRef;
     397             : 
     398             : } // end namespace object
     399             : 
     400             : namespace IndexedInstrProf {
     401             : 
     402             : uint64_t ComputeHash(StringRef K);
     403             : 
     404             : } // end namespace IndexedInstrProf
     405             : 
     406             : /// A symbol table used for function PGO name look-up with keys
     407             : /// (such as pointers, md5hash values) to the function. A function's
     408             : /// PGO name or name's md5hash are used in retrieving the profile
     409             : /// data of the function. See \c getPGOFuncName() method for details
     410             : /// on how PGO name is formed.
     411        3575 : class InstrProfSymtab {
     412             : public:
     413             :   using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
     414             : 
     415             : private:
     416             :   StringRef Data;
     417             :   uint64_t Address = 0;
     418             :   // Unique name strings.
     419             :   StringSet<> NameTab;
     420             :   // A map from MD5 keys to function name strings.
     421             :   std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
     422             :   // A map from MD5 keys to function define. We only populate this map
     423             :   // when build the Symtab from a Module.
     424             :   std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
     425             :   // A map from function runtime address to function name MD5 hash.
     426             :   // This map is only populated and used by raw instr profile reader.
     427             :   AddrHashMap AddrToMD5Map;
     428             : 
     429             : public:
     430        4326 :   InstrProfSymtab() = default; 
     431             : 
     432             :   /// Create InstrProfSymtab from an object file section which
     433             :   /// contains function PGO names. When section may contain raw
     434             :   /// string data or string data in compressed form. This method
     435             :   /// only initialize the symtab with reference to the data and
     436             :   /// the section base address. The decompression will be delayed
     437             :   /// until before it is used. See also \c create(StringRef) method.
     438             :   Error create(object::SectionRef &Section);
     439             : 
     440             :   /// This interface is used by reader of CoverageMapping test
     441             :   /// format.
     442             :   inline Error create(StringRef D, uint64_t BaseAddr);
     443             : 
     444             :   /// \c NameStrings is a string composed of one of more sub-strings
     445             :   ///  encoded in the format described in \c collectPGOFuncNameStrings.
     446             :   /// This method is a wrapper to \c readPGOFuncNameStrings method.
     447             :   inline Error create(StringRef NameStrings);
     448             : 
     449             :   /// A wrapper interface to populate the PGO symtab with functions
     450             :   /// decls from module \c M. This interface is used by transformation
     451             :   /// passes such as indirect function call promotion. Variable \c InLTO
     452             :   /// indicates if this is called from LTO optimization passes.
     453             :   Error create(Module &M, bool InLTO = false);
     454             : 
     455             :   /// Create InstrProfSymtab from a set of names iteratable from
     456             :   /// \p IterRange. This interface is used by IndexedProfReader.
     457             :   template <typename NameIterRange> Error create(const NameIterRange &IterRange);
     458             : 
     459             :   // If the symtab is created by a series of calls to \c addFuncName, \c
     460             :   // finalizeSymtab needs to be called before looking up function names.
     461             :   // This is required because the underlying map is a vector (for space
     462             :   // efficiency) which needs to be sorted.
     463             :   inline void finalizeSymtab();
     464             : 
     465             :   /// Update the symtab by adding \p FuncName to the table. This interface
     466             :   /// is used by the raw and text profile readers.
     467        1602 :   Error addFuncName(StringRef FuncName) {
     468        1602 :     if (FuncName.empty())
     469           2 :       return make_error<InstrProfError>(instrprof_error::malformed);
     470        3200 :     auto Ins = NameTab.insert(FuncName);
     471        1600 :     if (Ins.second)
     472        6188 :       MD5NameMap.push_back(std::make_pair(
     473        3094 :           IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
     474        4800 :     return Error::success();
     475             :   }
     476             : 
     477             :   /// Map a function address to its name's MD5 hash. This interface
     478             :   /// is only used by the raw profiler reader.
     479             :   void mapAddress(uint64_t Addr, uint64_t MD5Val) {
     480         132 :     AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
     481             :   }
     482             : 
     483           0 :   AddrHashMap &getAddrHashMap() { return AddrToMD5Map; }
     484             : 
     485             :   /// Return function's PGO name from the function name's symbol
     486             :   /// address in the object file. If an error occurs, return
     487             :   /// an empty string.
     488             :   StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
     489             : 
     490             :   /// Return function's PGO name from the name's md5 hash value.
     491             :   /// If not found, return an empty string.
     492             :   inline StringRef getFuncName(uint64_t FuncMD5Hash);
     493             : 
     494             :   /// Return function from the name's md5 hash. Return nullptr if not found.
     495             :   inline Function *getFunction(uint64_t FuncMD5Hash);
     496             : 
     497             :   /// Return the function's original assembly name by stripping off
     498             :   /// the prefix attached (to symbols with priviate linkage). For
     499             :   /// global functions, it returns the same string as getFuncName.
     500             :   inline StringRef getOrigFuncName(uint64_t FuncMD5Hash);
     501             : 
     502             :   /// Return the name section data.
     503             :   inline StringRef getNameData() const { return Data; }
     504             : };
     505             : 
     506             : Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
     507          70 :   Data = D;
     508          70 :   Address = BaseAddr;
     509         210 :   return Error::success();
     510             : }
     511             : 
     512             : Error InstrProfSymtab::create(StringRef NameStrings) {
     513          44 :   return readPGOFuncNameStrings(NameStrings, *this);
     514             : }
     515             : 
     516             : template <typename NameIterRange>
     517           4 : Error InstrProfSymtab::create(const NameIterRange &IterRange) {
     518          40 :   for (auto Name : IterRange)
     519          57 :     if (Error E = addFuncName(Name))
     520           0 :       return E;
     521             : 
     522           4 :   finalizeSymtab();
     523          12 :   return Error::success();
     524             : }
     525             : 
     526         892 : void InstrProfSymtab::finalizeSymtab() {
     527        2676 :   std::sort(MD5NameMap.begin(), MD5NameMap.end(), less_first());
     528        2676 :   std::sort(MD5FuncMap.begin(), MD5FuncMap.end(), less_first());
     529        2676 :   std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first());
     530        4460 :   AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
     531        3568 :                      AddrToMD5Map.end());
     532         892 : }
     533             : 
     534         329 : StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) {
     535             :   auto Result =
     536             :       std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash,
     537             :                        [](const std::pair<uint64_t, std::string> &LHS,
     538        2388 :                           uint64_t RHS) { return LHS.first < RHS; });
     539         987 :   if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
     540         325 :     return Result->second;
     541           4 :   return StringRef();
     542             : }
     543             : 
     544          41 : Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
     545             :   auto Result =
     546             :       std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash,
     547             :                        [](const std::pair<uint64_t, Function*> &LHS,
     548         270 :                           uint64_t RHS) { return LHS.first < RHS; });
     549         123 :   if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
     550          39 :     return Result->second;
     551             :   return nullptr;
     552             : }
     553             : 
     554             : // See also getPGOFuncName implementation. These two need to be
     555             : // matched.
     556          24 : StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) {
     557          24 :   StringRef PGOName = getFuncName(FuncMD5Hash);
     558          24 :   size_t S = PGOName.find_first_of(':');
     559          24 :   if (S == StringRef::npos)
     560          12 :     return PGOName;
     561          12 :   return PGOName.drop_front(S + 1);
     562             : }
     563             : 
     564         539 : struct InstrProfValueSiteRecord {
     565             :   /// Value profiling data pairs at a given value site.
     566             :   std::list<InstrProfValueData> ValueData;
     567             : 
     568         141 :   InstrProfValueSiteRecord() { ValueData.clear(); }
     569             :   template <class InputIterator>
     570         140 :   InstrProfValueSiteRecord(InputIterator F, InputIterator L)
     571         560 :       : ValueData(F, L) {}
     572             : 
     573             :   /// Sort ValueData ascending by Value
     574             :   void sortByTargetValues() {
     575          32 :     ValueData.sort(
     576             :         [](const InstrProfValueData &left, const InstrProfValueData &right) {
     577        4108 :           return left.Value < right.Value;
     578             :         });
     579             :   }
     580             :   /// Sort ValueData Descending by Count
     581             :   inline void sortByCount();
     582             : 
     583             :   /// Merge data from another InstrProfValueSiteRecord
     584             :   /// Optionally scale merged counts by \p Weight.
     585             :   void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
     586             :              function_ref<void(instrprof_error)> Warn);
     587             :   /// Scale up value profile data counts.
     588             :   void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
     589             : };
     590             : 
     591             : /// Profiling information for a single function.
     592       11187 : struct InstrProfRecord {
     593             :   std::vector<uint64_t> Counts;
     594             : 
     595        3129 :   InstrProfRecord() = default;
     596        2313 :   InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
     597        2670 :   InstrProfRecord(InstrProfRecord &&) = default;
     598         420 :   InstrProfRecord(const InstrProfRecord &RHS)
     599         420 :       : Counts(RHS.Counts),
     600             :         ValueData(RHS.ValueData
     601          40 :                       ? llvm::make_unique<ValueProfData>(*RHS.ValueData)
     602        1280 :                       : nullptr) {}
     603             :   InstrProfRecord &operator=(InstrProfRecord &&) = default;
     604         134 :   InstrProfRecord &operator=(const InstrProfRecord &RHS) {
     605         134 :     Counts = RHS.Counts;
     606         268 :     if (!RHS.ValueData) {
     607         132 :       ValueData = nullptr;
     608             :       return *this;
     609             :     }
     610           4 :     if (!ValueData)
     611           8 :       ValueData = llvm::make_unique<ValueProfData>(*RHS.ValueData);
     612             :     else
     613           0 :       *ValueData = *RHS.ValueData;
     614             :     return *this;
     615             :   }
     616             : 
     617             :   using ValueMapType = std::vector<std::pair<uint64_t, uint64_t>>;
     618             : 
     619             :   /// Return the number of value profile kinds with non-zero number
     620             :   /// of profile sites.
     621             :   inline uint32_t getNumValueKinds() const;
     622             :   /// Return the number of instrumented sites for ValueKind.
     623             :   inline uint32_t getNumValueSites(uint32_t ValueKind) const;
     624             : 
     625             :   /// Return the total number of ValueData for ValueKind.
     626             :   inline uint32_t getNumValueData(uint32_t ValueKind) const;
     627             : 
     628             :   /// Return the number of value data collected for ValueKind at profiling
     629             :   /// site: Site.
     630             :   inline uint32_t getNumValueDataForSite(uint32_t ValueKind,
     631             :                                          uint32_t Site) const;
     632             : 
     633             :   /// Return the array of profiled values at \p Site. If \p TotalC
     634             :   /// is not null, the total count of all target values at this site
     635             :   /// will be stored in \c *TotalC.
     636             :   inline std::unique_ptr<InstrProfValueData[]>
     637             :   getValueForSite(uint32_t ValueKind, uint32_t Site,
     638             :                   uint64_t *TotalC = nullptr) const;
     639             : 
     640             :   /// Get the target value/counts of kind \p ValueKind collected at site
     641             :   /// \p Site and store the result in array \p Dest. Return the total
     642             :   /// counts of all target values at this site.
     643             :   inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,
     644             :                                   uint32_t Site) const;
     645             : 
     646             :   /// Reserve space for NumValueSites sites.
     647             :   inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
     648             : 
     649             :   /// Add ValueData for ValueKind at value Site.
     650             :   void addValueData(uint32_t ValueKind, uint32_t Site,
     651             :                     InstrProfValueData *VData, uint32_t N,
     652             :                     ValueMapType *ValueMap);
     653             : 
     654             :   /// Merge the counts in \p Other into this one.
     655             :   /// Optionally scale merged counts by \p Weight.
     656             :   void merge(InstrProfRecord &Other, uint64_t Weight,
     657             :              function_ref<void(instrprof_error)> Warn);
     658             : 
     659             :   /// Scale up profile counts (including value profile data) by
     660             :   /// \p Weight.
     661             :   void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
     662             : 
     663             :   /// Sort value profile data (per site) by count.
     664         422 :   void sortValueData() {
     665        1266 :     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
     666        2603 :       for (auto &SR : getValueSitesForKind(Kind))
     667          71 :         SR.sortByCount();
     668         422 :   }
     669             : 
     670             :   /// Clear value data entries and edge counters.
     671             :   void Clear() {
     672         462 :     Counts.clear();
     673         231 :     clearValueData();
     674             :   }
     675             : 
     676             :   /// Clear value data entries
     677         554 :   void clearValueData() { ValueData = nullptr; }
     678             : 
     679             : private:
     680         296 :   struct ValueProfData {
     681             :     std::vector<InstrProfValueSiteRecord> IndirectCallSites;
     682             :     std::vector<InstrProfValueSiteRecord> MemOPSizes;
     683             :   };
     684             :   std::unique_ptr<ValueProfData> ValueData;
     685             : 
     686             :   MutableArrayRef<InstrProfValueSiteRecord>
     687             :   getValueSitesForKind(uint32_t ValueKind) {
     688             :     // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
     689             :     // implemented in LLVM) to call the const overload of this function, then
     690             :     // cast away the constness from the result.
     691             :     auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
     692         870 :         ValueKind);
     693             :     return makeMutableArrayRef(
     694        1740 :         const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
     695             :   }
     696             :   ArrayRef<InstrProfValueSiteRecord>
     697        4447 :   getValueSitesForKind(uint32_t ValueKind) const {
     698        8894 :     if (!ValueData)
     699        3734 :       return None;
     700         713 :     switch (ValueKind) {
     701         532 :     case IPVK_IndirectCallTarget:
     702        1596 :       return ValueData->IndirectCallSites;
     703         181 :     case IPVK_MemOPSize:
     704         543 :       return ValueData->MemOPSizes;
     705           0 :     default:
     706           0 :       llvm_unreachable("Unknown value kind!");
     707             :     }
     708             :   }
     709             : 
     710             :   std::vector<InstrProfValueSiteRecord> &
     711         262 :   getOrCreateValueSitesForKind(uint32_t ValueKind) {
     712         524 :     if (!ValueData)
     713         189 :       ValueData = llvm::make_unique<ValueProfData>();
     714         262 :     switch (ValueKind) {
     715         241 :     case IPVK_IndirectCallTarget:
     716         482 :       return ValueData->IndirectCallSites;
     717          21 :     case IPVK_MemOPSize:
     718          42 :       return ValueData->MemOPSizes;
     719           0 :     default:
     720           0 :       llvm_unreachable("Unknown value kind!");
     721             :     }
     722             :   }
     723             : 
     724             :   // Map indirect call target name hash to name string.
     725             :   uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
     726             :                       ValueMapType *HashKeys);
     727             : 
     728             :   // Merge Value Profile data from Src record to this record for ValueKind.
     729             :   // Scale merged value counts by \p Weight.
     730             :   void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
     731             :                           uint64_t Weight,
     732             :                           function_ref<void(instrprof_error)> Warn);
     733             : 
     734             :   // Scale up value profile data count.
     735             :   void scaleValueProfData(uint32_t ValueKind, uint64_t Weight,
     736             :                           function_ref<void(instrprof_error)> Warn);
     737             : };
     738             : 
     739        1680 : struct NamedInstrProfRecord : InstrProfRecord {
     740             :   StringRef Name;
     741             :   uint64_t Hash;
     742             : 
     743        1316 :   NamedInstrProfRecord() = default;
     744             :   NamedInstrProfRecord(StringRef Name, uint64_t Hash,
     745             :                        std::vector<uint64_t> Counts)
     746        3068 :       : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
     747             : };
     748             : 
     749             : uint32_t InstrProfRecord::getNumValueKinds() const {
     750         341 :   uint32_t NumValueKinds = 0;
     751        1133 :   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
     752         726 :     NumValueKinds += !(getValueSitesForKind(Kind).empty());
     753             :   return NumValueKinds;
     754             : }
     755             : 
     756          42 : uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
     757          42 :   uint32_t N = 0;
     758         206 :   for (auto &SR : getValueSitesForKind(ValueKind))
     759         244 :     N += SR.ValueData.size();
     760          42 :   return N;
     761             : }
     762             : 
     763             : uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
     764        2454 :   return getValueSitesForKind(ValueKind).size();
     765             : }
     766             : 
     767             : uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind,
     768             :                                                  uint32_t Site) const {
     769         661 :   return getValueSitesForKind(ValueKind)[Site].ValueData.size();
     770             : }
     771             : 
     772             : std::unique_ptr<InstrProfValueData[]>
     773          67 : InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site,
     774             :                                  uint64_t *TotalC) const {
     775             :   uint64_t Dummy;
     776          67 :   uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC);
     777          67 :   uint32_t N = getNumValueDataForSite(ValueKind, Site);
     778          67 :   if (N == 0) {
     779           8 :     TotalCount = 0;
     780             :     return std::unique_ptr<InstrProfValueData[]>(nullptr);
     781             :   }
     782             : 
     783          59 :   auto VD = llvm::make_unique<InstrProfValueData[]>(N);
     784          59 :   TotalCount = getValueForSite(VD.get(), ValueKind, Site);
     785             : 
     786          59 :   return VD;
     787             : }
     788             : 
     789         130 : uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[],
     790             :                                           uint32_t ValueKind,
     791             :                                           uint32_t Site) const {
     792         130 :   uint32_t I = 0;
     793         130 :   uint64_t TotalCount = 0;
     794        1914 :   for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
     795        1394 :     Dest[I].Value = V.Value;
     796        1394 :     Dest[I].Count = V.Count;
     797        2788 :     TotalCount = SaturatingAdd(TotalCount, V.Count);
     798        1394 :     I++;
     799             :   }
     800         130 :   return TotalCount;
     801             : }
     802             : 
     803          45 : void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
     804          45 :   if (!NumValueSites)
     805             :     return;
     806          69 :   getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
     807             : }
     808             : 
     809             : inline support::endianness getHostEndianness() {
     810             :   return sys::IsLittleEndianHost ? support::little : support::big;
     811             : }
     812             : 
     813             : // Include definitions for value profile data
     814             : #define INSTR_PROF_VALUE_PROF_DATA
     815             : #include "llvm/ProfileData/InstrProfData.inc"
     816             : 
     817          71 : void InstrProfValueSiteRecord::sortByCount() {
     818          71 :   ValueData.sort(
     819             :       [](const InstrProfValueData &left, const InstrProfValueData &right) {
     820        6751 :         return left.Count > right.Count;
     821             :       });
     822             :   // Now truncate
     823          71 :   size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
     824         142 :   if (ValueData.size() > max_s)
     825           2 :     ValueData.resize(max_s);
     826          71 : }
     827             : 
     828             : namespace IndexedInstrProf {
     829             : 
     830             : enum class HashT : uint32_t {
     831             :   MD5,
     832             :   Last = MD5
     833             : };
     834             : 
     835             : inline uint64_t ComputeHash(HashT Type, StringRef K) {
     836         477 :   switch (Type) {
     837         477 :   case HashT::MD5:
     838        3032 :     return MD5Hash(K);
     839             :   }
     840           0 :   llvm_unreachable("Unhandled hash type");
     841             : }
     842             : 
     843             : const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
     844             : 
     845             : enum ProfVersion {
     846             :   // Version 1 is the first version. In this version, the value of
     847             :   // a key/value pair can only include profile data of a single function.
     848             :   // Due to this restriction, the number of block counters for a given
     849             :   // function is not recorded but derived from the length of the value.
     850             :   Version1 = 1,
     851             :   // The version 2 format supports recording profile data of multiple
     852             :   // functions which share the same key in one value field. To support this,
     853             :   // the number block counters is recorded as an uint64_t field right after the
     854             :   // function structural hash.
     855             :   Version2 = 2,
     856             :   // Version 3 supports value profile data. The value profile data is expected
     857             :   // to follow the block counter profile data.
     858             :   Version3 = 3,
     859             :   // In this version, profile summary data \c IndexedInstrProf::Summary is
     860             :   // stored after the profile header.
     861             :   Version4 = 4,
     862             :   // The current version is 4.
     863             :   CurrentVersion = INSTR_PROF_INDEX_VERSION
     864             : };
     865             : const uint64_t Version = ProfVersion::CurrentVersion;
     866             : 
     867             : const HashT HashType = HashT::MD5;
     868             : 
     869        2557 : inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); }
     870             : 
     871             : // This structure defines the file header of the LLVM profile
     872             : // data file in indexed-format.
     873             : struct Header {
     874             :   uint64_t Magic;
     875             :   uint64_t Version;
     876             :   uint64_t Unused; // Becomes unused since version 4
     877             :   uint64_t HashType;
     878             :   uint64_t HashOffset;
     879             : };
     880             : 
     881             : // Profile summary data recorded in the profile data file in indexed
     882             : // format. It is introduced in version 4. The summary data follows
     883             : // right after the profile file header.
     884             : struct Summary {
     885             :   struct Entry {
     886             :     uint64_t Cutoff; ///< The required percentile of total execution count.
     887             :     uint64_t
     888             :         MinBlockCount;  ///< The minimum execution count for this percentile.
     889             :     uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
     890             :   };
     891             :   // The field kind enumerator to assigned value mapping should remain
     892             :   // unchanged  when a new kind is added or an old kind gets deleted in
     893             :   // the future.
     894             :   enum SummaryFieldKind {
     895             :     /// The total number of functions instrumented.
     896             :     TotalNumFunctions = 0,
     897             :     /// Total number of instrumented blocks/edges.
     898             :     TotalNumBlocks = 1,
     899             :     /// The maximal execution count among all functions.
     900             :     /// This field does not exist for profile data from IR based
     901             :     /// instrumentation.
     902             :     MaxFunctionCount = 2,
     903             :     /// Max block count of the program.
     904             :     MaxBlockCount = 3,
     905             :     /// Max internal block count of the program (excluding entry blocks).
     906             :     MaxInternalBlockCount = 4,
     907             :     /// The sum of all instrumented block counts.
     908             :     TotalBlockCount = 5,
     909             :     NumKinds = TotalBlockCount + 1
     910             :   };
     911             : 
     912             :   // The number of summmary fields following the summary header.
     913             :   uint64_t NumSummaryFields;
     914             :   // The number of Cutoff Entries (Summary::Entry) following summary fields.
     915             :   uint64_t NumCutoffEntries;
     916             : 
     917             :   Summary() = delete;
     918         519 :   Summary(uint32_t Size) { memset(this, 0, Size); }
     919             : 
     920         519 :   void operator delete(void *ptr) { ::operator delete(ptr); }
     921             : 
     922             :   static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) {
     923         519 :     return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
     924         519 :            NumSumFields * sizeof(uint64_t);
     925             :   }
     926             : 
     927             :   const uint64_t *getSummaryDataBase() const {
     928             :     return reinterpret_cast<const uint64_t *>(this + 1);
     929             :   }
     930             : 
     931             :   uint64_t *getSummaryDataBase() {
     932             :     return reinterpret_cast<uint64_t *>(this + 1);
     933             :   }
     934             : 
     935             :   const Entry *getCutoffEntryBase() const {
     936             :     return reinterpret_cast<const Entry *>(
     937        5688 :         &getSummaryDataBase()[NumSummaryFields]);
     938             :   }
     939             : 
     940             :   Entry *getCutoffEntryBase() {
     941        3654 :     return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
     942             :   }
     943             : 
     944             :   uint64_t get(SummaryFieldKind K) const {
     945        1896 :     return getSummaryDataBase()[K];
     946             :   }
     947             : 
     948             :   void set(SummaryFieldKind K, uint64_t V) {
     949        1218 :     getSummaryDataBase()[K] = V;
     950             :   }
     951             : 
     952        5688 :   const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
     953             : 
     954             :   void setEntry(uint32_t I, const ProfileSummaryEntry &E) {
     955        3654 :     Entry &ER = getCutoffEntryBase()[I];
     956        3654 :     ER.Cutoff = E.Cutoff;
     957        3654 :     ER.MinBlockCount = E.MinCount;
     958        3654 :     ER.NumBlocks = E.NumCounts;
     959             :   }
     960             : };
     961             : 
     962         519 : inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
     963         519 :   return std::unique_ptr<Summary>(new (::operator new(TotalSize))
     964        1557 :                                       Summary(TotalSize));
     965             : }
     966             : 
     967             : } // end namespace IndexedInstrProf
     968             : 
     969             : namespace RawInstrProf {
     970             : 
     971             : // Version 1: First version
     972             : // Version 2: Added value profile data section. Per-function control data
     973             : // struct has more fields to describe value profile information.
     974             : // Version 3: Compressed name section support. Function PGO name reference
     975             : // from control data struct is changed from raw pointer to Name's MD5 value.
     976             : // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
     977             : // raw header.
     978             : const uint64_t Version = INSTR_PROF_RAW_VERSION;
     979             : 
     980             : template <class IntPtrT> inline uint64_t getMagic();
     981             : template <> inline uint64_t getMagic<uint64_t>() {
     982             :   return INSTR_PROF_RAW_MAGIC_64;
     983             : }
     984             : 
     985             : template <> inline uint64_t getMagic<uint32_t>() {
     986             :   return INSTR_PROF_RAW_MAGIC_32;
     987             : }
     988             : 
     989             : // Per-function profile data header/control structure.
     990             : // The definition should match the structure defined in
     991             : // compiler-rt/lib/profile/InstrProfiling.h.
     992             : // It should also match the synthesized type in
     993             : // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
     994             : template <class IntPtrT> struct LLVM_ALIGNAS(8) ProfileData {
     995             :   #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
     996             :   #include "llvm/ProfileData/InstrProfData.inc"
     997             : };
     998             : 
     999             : // File header structure of the LLVM profile data in raw format.
    1000             : // The definition should match the header referenced in
    1001             : // compiler-rt/lib/profile/InstrProfilingFile.c  and
    1002             : // InstrProfilingBuffer.c.
    1003             : struct Header {
    1004             : #define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
    1005             : #include "llvm/ProfileData/InstrProfData.inc"
    1006             : };
    1007             : 
    1008             : } // end namespace RawInstrProf
    1009             : 
    1010             : // Parse MemOP Size range option.
    1011             : void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
    1012             :                                  int64_t &RangeLast);
    1013             : 
    1014             : } // end namespace llvm
    1015             : 
    1016             : #endif // LLVM_PROFILEDATA_INSTRPROF_H

Generated by: LCOV version 1.13