| File: | llvm/lib/ProfileData/InstrProf.cpp |
| Warning: | line 498, column 24 Dereference of null pointer (loaded from variable 'P') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- InstrProf.cpp - Instrumented profiling format support --------------===// | ||||
| 2 | // | ||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
| 6 | // | ||||
| 7 | //===----------------------------------------------------------------------===// | ||||
| 8 | // | ||||
| 9 | // This file contains support for clang's instrumentation based PGO and | ||||
| 10 | // coverage. | ||||
| 11 | // | ||||
| 12 | //===----------------------------------------------------------------------===// | ||||
| 13 | |||||
| 14 | #include "llvm/ProfileData/InstrProf.h" | ||||
| 15 | #include "llvm/ADT/ArrayRef.h" | ||||
| 16 | #include "llvm/ADT/SmallString.h" | ||||
| 17 | #include "llvm/ADT/SmallVector.h" | ||||
| 18 | #include "llvm/ADT/StringExtras.h" | ||||
| 19 | #include "llvm/ADT/StringRef.h" | ||||
| 20 | #include "llvm/ADT/Triple.h" | ||||
| 21 | #include "llvm/Config/config.h" | ||||
| 22 | #include "llvm/IR/Constant.h" | ||||
| 23 | #include "llvm/IR/Constants.h" | ||||
| 24 | #include "llvm/IR/Function.h" | ||||
| 25 | #include "llvm/IR/GlobalValue.h" | ||||
| 26 | #include "llvm/IR/GlobalVariable.h" | ||||
| 27 | #include "llvm/IR/Instruction.h" | ||||
| 28 | #include "llvm/IR/LLVMContext.h" | ||||
| 29 | #include "llvm/IR/MDBuilder.h" | ||||
| 30 | #include "llvm/IR/Metadata.h" | ||||
| 31 | #include "llvm/IR/Module.h" | ||||
| 32 | #include "llvm/IR/Type.h" | ||||
| 33 | #include "llvm/ProfileData/InstrProfReader.h" | ||||
| 34 | #include "llvm/Support/Casting.h" | ||||
| 35 | #include "llvm/Support/CommandLine.h" | ||||
| 36 | #include "llvm/Support/Compiler.h" | ||||
| 37 | #include "llvm/Support/Compression.h" | ||||
| 38 | #include "llvm/Support/Endian.h" | ||||
| 39 | #include "llvm/Support/Error.h" | ||||
| 40 | #include "llvm/Support/ErrorHandling.h" | ||||
| 41 | #include "llvm/Support/LEB128.h" | ||||
| 42 | #include "llvm/Support/ManagedStatic.h" | ||||
| 43 | #include "llvm/Support/MathExtras.h" | ||||
| 44 | #include "llvm/Support/Path.h" | ||||
| 45 | #include "llvm/Support/SwapByteOrder.h" | ||||
| 46 | #include <algorithm> | ||||
| 47 | #include <cassert> | ||||
| 48 | #include <cstddef> | ||||
| 49 | #include <cstdint> | ||||
| 50 | #include <cstring> | ||||
| 51 | #include <memory> | ||||
| 52 | #include <string> | ||||
| 53 | #include <system_error> | ||||
| 54 | #include <utility> | ||||
| 55 | #include <vector> | ||||
| 56 | |||||
| 57 | using namespace llvm; | ||||
| 58 | |||||
| 59 | static cl::opt<bool> StaticFuncFullModulePrefix( | ||||
| 60 | "static-func-full-module-prefix", cl::init(true), cl::Hidden, | ||||
| 61 | cl::desc("Use full module build paths in the profile counter names for " | ||||
| 62 | "static functions.")); | ||||
| 63 | |||||
| 64 | // This option is tailored to users that have different top-level directory in | ||||
| 65 | // profile-gen and profile-use compilation. Users need to specific the number | ||||
| 66 | // of levels to strip. A value larger than the number of directories in the | ||||
| 67 | // source file will strip all the directory names and only leave the basename. | ||||
| 68 | // | ||||
| 69 | // Note current ThinLTO module importing for the indirect-calls assumes | ||||
| 70 | // the source directory name not being stripped. A non-zero option value here | ||||
| 71 | // can potentially prevent some inter-module indirect-call-promotions. | ||||
| 72 | static cl::opt<unsigned> StaticFuncStripDirNamePrefix( | ||||
| 73 | "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, | ||||
| 74 | cl::desc("Strip specified level of directory name from source path in " | ||||
| 75 | "the profile counter name for static functions.")); | ||||
| 76 | |||||
| 77 | static std::string getInstrProfErrString(instrprof_error Err) { | ||||
| 78 | switch (Err) { | ||||
| 79 | case instrprof_error::success: | ||||
| 80 | return "success"; | ||||
| 81 | case instrprof_error::eof: | ||||
| 82 | return "end of File"; | ||||
| 83 | case instrprof_error::unrecognized_format: | ||||
| 84 | return "unrecognized instrumentation profile encoding format"; | ||||
| 85 | case instrprof_error::bad_magic: | ||||
| 86 | return "invalid instrumentation profile data (bad magic)"; | ||||
| 87 | case instrprof_error::bad_header: | ||||
| 88 | return "invalid instrumentation profile data (file header is corrupt)"; | ||||
| 89 | case instrprof_error::unsupported_version: | ||||
| 90 | return "unsupported instrumentation profile format version"; | ||||
| 91 | case instrprof_error::unsupported_hash_type: | ||||
| 92 | return "unsupported instrumentation profile hash type"; | ||||
| 93 | case instrprof_error::too_large: | ||||
| 94 | return "too much profile data"; | ||||
| 95 | case instrprof_error::truncated: | ||||
| 96 | return "truncated profile data"; | ||||
| 97 | case instrprof_error::malformed: | ||||
| 98 | return "malformed instrumentation profile data"; | ||||
| 99 | case instrprof_error::invalid_prof: | ||||
| 100 | return "invalid profile created. Please file a bug " | ||||
| 101 | "at: " BUG_REPORT_URL"https://bugs.llvm.org/" | ||||
| 102 | " and include the profraw files that caused this error."; | ||||
| 103 | case instrprof_error::unknown_function: | ||||
| 104 | return "no profile data available for function"; | ||||
| 105 | case instrprof_error::hash_mismatch: | ||||
| 106 | return "function control flow change detected (hash mismatch)"; | ||||
| 107 | case instrprof_error::count_mismatch: | ||||
| 108 | return "function basic block count change detected (counter mismatch)"; | ||||
| 109 | case instrprof_error::counter_overflow: | ||||
| 110 | return "counter overflow"; | ||||
| 111 | case instrprof_error::value_site_count_mismatch: | ||||
| 112 | return "function value site count change detected (counter mismatch)"; | ||||
| 113 | case instrprof_error::compress_failed: | ||||
| 114 | return "failed to compress data (zlib)"; | ||||
| 115 | case instrprof_error::uncompress_failed: | ||||
| 116 | return "failed to uncompress data (zlib)"; | ||||
| 117 | case instrprof_error::empty_raw_profile: | ||||
| 118 | return "empty raw profile file"; | ||||
| 119 | case instrprof_error::zlib_unavailable: | ||||
| 120 | return "profile uses zlib compression but the profile reader was built " | ||||
| 121 | "without zlib support"; | ||||
| 122 | } | ||||
| 123 | llvm_unreachable("A value of instrprof_error has no message.")::llvm::llvm_unreachable_internal("A value of instrprof_error has no message." , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 123); | ||||
| 124 | } | ||||
| 125 | |||||
| 126 | namespace { | ||||
| 127 | |||||
| 128 | // FIXME: This class is only here to support the transition to llvm::Error. It | ||||
| 129 | // will be removed once this transition is complete. Clients should prefer to | ||||
| 130 | // deal with the Error value directly, rather than converting to error_code. | ||||
| 131 | class InstrProfErrorCategoryType : public std::error_category { | ||||
| 132 | const char *name() const noexcept override { return "llvm.instrprof"; } | ||||
| 133 | |||||
| 134 | std::string message(int IE) const override { | ||||
| 135 | return getInstrProfErrString(static_cast<instrprof_error>(IE)); | ||||
| 136 | } | ||||
| 137 | }; | ||||
| 138 | |||||
| 139 | } // end anonymous namespace | ||||
| 140 | |||||
| 141 | static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory; | ||||
| 142 | |||||
| 143 | const std::error_category &llvm::instrprof_category() { | ||||
| 144 | return *ErrorCategory; | ||||
| 145 | } | ||||
| 146 | |||||
| 147 | namespace { | ||||
| 148 | |||||
| 149 | const char *InstrProfSectNameCommon[] = { | ||||
| 150 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ | ||||
| 151 | SectNameCommon, | ||||
| 152 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
| 153 | }; | ||||
| 154 | |||||
| 155 | const char *InstrProfSectNameCoff[] = { | ||||
| 156 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ | ||||
| 157 | SectNameCoff, | ||||
| 158 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
| 159 | }; | ||||
| 160 | |||||
| 161 | const char *InstrProfSectNamePrefix[] = { | ||||
| 162 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ | ||||
| 163 | Prefix, | ||||
| 164 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
| 165 | }; | ||||
| 166 | |||||
| 167 | } // namespace | ||||
| 168 | |||||
| 169 | namespace llvm { | ||||
| 170 | |||||
| 171 | cl::opt<bool> DoInstrProfNameCompression( | ||||
| 172 | "enable-name-compression", | ||||
| 173 | cl::desc("Enable name/filename string compression"), cl::init(true)); | ||||
| 174 | |||||
| 175 | std::string getInstrProfSectionName(InstrProfSectKind IPSK, | ||||
| 176 | Triple::ObjectFormatType OF, | ||||
| 177 | bool AddSegmentInfo) { | ||||
| 178 | std::string SectName; | ||||
| 179 | |||||
| 180 | if (OF == Triple::MachO && AddSegmentInfo) | ||||
| 181 | SectName = InstrProfSectNamePrefix[IPSK]; | ||||
| 182 | |||||
| 183 | if (OF == Triple::COFF) | ||||
| 184 | SectName += InstrProfSectNameCoff[IPSK]; | ||||
| 185 | else | ||||
| 186 | SectName += InstrProfSectNameCommon[IPSK]; | ||||
| 187 | |||||
| 188 | if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo) | ||||
| 189 | SectName += ",regular,live_support"; | ||||
| 190 | |||||
| 191 | return SectName; | ||||
| 192 | } | ||||
| 193 | |||||
| 194 | void SoftInstrProfErrors::addError(instrprof_error IE) { | ||||
| 195 | if (IE == instrprof_error::success) | ||||
| 196 | return; | ||||
| 197 | |||||
| 198 | if (FirstError == instrprof_error::success) | ||||
| 199 | FirstError = IE; | ||||
| 200 | |||||
| 201 | switch (IE) { | ||||
| 202 | case instrprof_error::hash_mismatch: | ||||
| 203 | ++NumHashMismatches; | ||||
| 204 | break; | ||||
| 205 | case instrprof_error::count_mismatch: | ||||
| 206 | ++NumCountMismatches; | ||||
| 207 | break; | ||||
| 208 | case instrprof_error::counter_overflow: | ||||
| 209 | ++NumCounterOverflows; | ||||
| 210 | break; | ||||
| 211 | case instrprof_error::value_site_count_mismatch: | ||||
| 212 | ++NumValueSiteCountMismatches; | ||||
| 213 | break; | ||||
| 214 | default: | ||||
| 215 | llvm_unreachable("Not a soft error")::llvm::llvm_unreachable_internal("Not a soft error", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 215); | ||||
| 216 | } | ||||
| 217 | } | ||||
| 218 | |||||
| 219 | std::string InstrProfError::message() const { | ||||
| 220 | return getInstrProfErrString(Err); | ||||
| 221 | } | ||||
| 222 | |||||
| 223 | char InstrProfError::ID = 0; | ||||
| 224 | |||||
| 225 | std::string getPGOFuncName(StringRef RawFuncName, | ||||
| 226 | GlobalValue::LinkageTypes Linkage, | ||||
| 227 | StringRef FileName, | ||||
| 228 | uint64_t Version LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__))) { | ||||
| 229 | return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); | ||||
| 230 | } | ||||
| 231 | |||||
| 232 | // Strip NumPrefix level of directory name from PathNameStr. If the number of | ||||
| 233 | // directory separators is less than NumPrefix, strip all the directories and | ||||
| 234 | // leave base file name only. | ||||
| 235 | static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { | ||||
| 236 | uint32_t Count = NumPrefix; | ||||
| 237 | uint32_t Pos = 0, LastPos = 0; | ||||
| 238 | for (auto & CI : PathNameStr) { | ||||
| 239 | ++Pos; | ||||
| 240 | if (llvm::sys::path::is_separator(CI)) { | ||||
| 241 | LastPos = Pos; | ||||
| 242 | --Count; | ||||
| 243 | } | ||||
| 244 | if (Count == 0) | ||||
| 245 | break; | ||||
| 246 | } | ||||
| 247 | return PathNameStr.substr(LastPos); | ||||
| 248 | } | ||||
| 249 | |||||
| 250 | // Return the PGOFuncName. This function has some special handling when called | ||||
| 251 | // in LTO optimization. The following only applies when calling in LTO passes | ||||
| 252 | // (when \c InLTO is true): LTO's internalization privatizes many global linkage | ||||
| 253 | // symbols. This happens after value profile annotation, but those internal | ||||
| 254 | // linkage functions should not have a source prefix. | ||||
| 255 | // Additionally, for ThinLTO mode, exported internal functions are promoted | ||||
| 256 | // and renamed. We need to ensure that the original internal PGO name is | ||||
| 257 | // used when computing the GUID that is compared against the profiled GUIDs. | ||||
| 258 | // To differentiate compiler generated internal symbols from original ones, | ||||
| 259 | // PGOFuncName meta data are created and attached to the original internal | ||||
| 260 | // symbols in the value profile annotation step | ||||
| 261 | // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta | ||||
| 262 | // data, its original linkage must be non-internal. | ||||
| 263 | std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { | ||||
| 264 | if (!InLTO) { | ||||
| 265 | StringRef FileName(F.getParent()->getSourceFileName()); | ||||
| 266 | uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1; | ||||
| 267 | if (StripLevel < StaticFuncStripDirNamePrefix) | ||||
| 268 | StripLevel = StaticFuncStripDirNamePrefix; | ||||
| 269 | if (StripLevel) | ||||
| 270 | FileName = stripDirPrefix(FileName, StripLevel); | ||||
| 271 | return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version); | ||||
| 272 | } | ||||
| 273 | |||||
| 274 | // In LTO mode (when InLTO is true), first check if there is a meta data. | ||||
| 275 | if (MDNode *MD = getPGOFuncNameMetadata(F)) { | ||||
| 276 | StringRef S = cast<MDString>(MD->getOperand(0))->getString(); | ||||
| 277 | return S.str(); | ||||
| 278 | } | ||||
| 279 | |||||
| 280 | // If there is no meta data, the function must be a global before the value | ||||
| 281 | // profile annotation pass. Its current linkage may be internal if it is | ||||
| 282 | // internalized in LTO mode. | ||||
| 283 | return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); | ||||
| 284 | } | ||||
| 285 | |||||
| 286 | StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { | ||||
| 287 | if (FileName.empty()) | ||||
| 288 | return PGOFuncName; | ||||
| 289 | // Drop the file name including ':'. See also getPGOFuncName. | ||||
| 290 | if (PGOFuncName.startswith(FileName)) | ||||
| 291 | PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1); | ||||
| 292 | return PGOFuncName; | ||||
| 293 | } | ||||
| 294 | |||||
| 295 | // \p FuncName is the string used as profile lookup key for the function. A | ||||
| 296 | // symbol is created to hold the name. Return the legalized symbol name. | ||||
| 297 | std::string getPGOFuncNameVarName(StringRef FuncName, | ||||
| 298 | GlobalValue::LinkageTypes Linkage) { | ||||
| 299 | std::string VarName = std::string(getInstrProfNameVarPrefix()); | ||||
| 300 | VarName += FuncName; | ||||
| 301 | |||||
| 302 | if (!GlobalValue::isLocalLinkage(Linkage)) | ||||
| 303 | return VarName; | ||||
| 304 | |||||
| 305 | // Now fix up illegal chars in local VarName that may upset the assembler. | ||||
| 306 | const char *InvalidChars = "-:<>/\"'"; | ||||
| 307 | size_t found = VarName.find_first_of(InvalidChars); | ||||
| 308 | while (found != std::string::npos) { | ||||
| 309 | VarName[found] = '_'; | ||||
| 310 | found = VarName.find_first_of(InvalidChars, found + 1); | ||||
| 311 | } | ||||
| 312 | return VarName; | ||||
| 313 | } | ||||
| 314 | |||||
| 315 | GlobalVariable *createPGOFuncNameVar(Module &M, | ||||
| 316 | GlobalValue::LinkageTypes Linkage, | ||||
| 317 | StringRef PGOFuncName) { | ||||
| 318 | // We generally want to match the function's linkage, but available_externally | ||||
| 319 | // and extern_weak both have the wrong semantics, and anything that doesn't | ||||
| 320 | // need to link across compilation units doesn't need to be visible at all. | ||||
| 321 | if (Linkage == GlobalValue::ExternalWeakLinkage) | ||||
| 322 | Linkage = GlobalValue::LinkOnceAnyLinkage; | ||||
| 323 | else if (Linkage == GlobalValue::AvailableExternallyLinkage) | ||||
| 324 | Linkage = GlobalValue::LinkOnceODRLinkage; | ||||
| 325 | else if (Linkage == GlobalValue::InternalLinkage || | ||||
| 326 | Linkage == GlobalValue::ExternalLinkage) | ||||
| 327 | Linkage = GlobalValue::PrivateLinkage; | ||||
| 328 | |||||
| 329 | auto *Value = | ||||
| 330 | ConstantDataArray::getString(M.getContext(), PGOFuncName, false); | ||||
| 331 | auto FuncNameVar = | ||||
| 332 | new GlobalVariable(M, Value->getType(), true, Linkage, Value, | ||||
| 333 | getPGOFuncNameVarName(PGOFuncName, Linkage)); | ||||
| 334 | |||||
| 335 | // Hide the symbol so that we correctly get a copy for each executable. | ||||
| 336 | if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) | ||||
| 337 | FuncNameVar->setVisibility(GlobalValue::HiddenVisibility); | ||||
| 338 | |||||
| 339 | return FuncNameVar; | ||||
| 340 | } | ||||
| 341 | |||||
| 342 | GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) { | ||||
| 343 | return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName); | ||||
| 344 | } | ||||
| 345 | |||||
| 346 | Error InstrProfSymtab::create(Module &M, bool InLTO) { | ||||
| 347 | for (Function &F : M) { | ||||
| 348 | // Function may not have a name: like using asm("") to overwrite the name. | ||||
| 349 | // Ignore in this case. | ||||
| 350 | if (!F.hasName()) | ||||
| 351 | continue; | ||||
| 352 | const std::string &PGOFuncName = getPGOFuncName(F, InLTO); | ||||
| 353 | if (Error E = addFuncName(PGOFuncName)) | ||||
| 354 | return E; | ||||
| 355 | MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); | ||||
| 356 | // In ThinLTO, local function may have been promoted to global and have | ||||
| 357 | // suffix ".llvm." added to the function name. We need to add the | ||||
| 358 | // stripped function name to the symbol table so that we can find a match | ||||
| 359 | // from profile. | ||||
| 360 | // | ||||
| 361 | // We may have other suffixes similar as ".llvm." which are needed to | ||||
| 362 | // be stripped before the matching, but ".__uniq." suffix which is used | ||||
| 363 | // to differentiate internal linkage functions in different modules | ||||
| 364 | // should be kept. Now this is the only suffix with the pattern ".xxx" | ||||
| 365 | // which is kept before matching. | ||||
| 366 | const std::string UniqSuffix = ".__uniq."; | ||||
| 367 | auto pos = PGOFuncName.find(UniqSuffix); | ||||
| 368 | // Search '.' after ".__uniq." if ".__uniq." exists, otherwise | ||||
| 369 | // search '.' from the beginning. | ||||
| 370 | if (pos != std::string::npos) | ||||
| 371 | pos += UniqSuffix.length(); | ||||
| 372 | else | ||||
| 373 | pos = 0; | ||||
| 374 | pos = PGOFuncName.find('.', pos); | ||||
| 375 | if (pos != std::string::npos && pos != 0) { | ||||
| 376 | const std::string &OtherFuncName = PGOFuncName.substr(0, pos); | ||||
| 377 | if (Error E = addFuncName(OtherFuncName)) | ||||
| 378 | return E; | ||||
| 379 | MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); | ||||
| 380 | } | ||||
| 381 | } | ||||
| 382 | Sorted = false; | ||||
| 383 | finalizeSymtab(); | ||||
| 384 | return Error::success(); | ||||
| 385 | } | ||||
| 386 | |||||
| 387 | uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { | ||||
| 388 | finalizeSymtab(); | ||||
| 389 | auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) { | ||||
| 390 | return A.first < Address; | ||||
| 391 | }); | ||||
| 392 | // Raw function pointer collected by value profiler may be from | ||||
| 393 | // external functions that are not instrumented. They won't have | ||||
| 394 | // mapping data to be used by the deserializer. Force the value to | ||||
| 395 | // be 0 in this case. | ||||
| 396 | if (It != AddrToMD5Map.end() && It->first == Address) | ||||
| 397 | return (uint64_t)It->second; | ||||
| 398 | return 0; | ||||
| 399 | } | ||||
| 400 | |||||
| 401 | Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs, | ||||
| 402 | bool doCompression, std::string &Result) { | ||||
| 403 | assert(!NameStrs.empty() && "No name data to emit")(static_cast <bool> (!NameStrs.empty() && "No name data to emit" ) ? void (0) : __assert_fail ("!NameStrs.empty() && \"No name data to emit\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 403, __extension__ __PRETTY_FUNCTION__)); | ||||
| 404 | |||||
| 405 | uint8_t Header[16], *P = Header; | ||||
| 406 | std::string UncompressedNameStrings = | ||||
| 407 | join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator()); | ||||
| 408 | |||||
| 409 | assert(StringRef(UncompressedNameStrings)(static_cast <bool> (StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)") ? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 411, __extension__ __PRETTY_FUNCTION__)) | ||||
| 410 | .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&(static_cast <bool> (StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)") ? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 411, __extension__ __PRETTY_FUNCTION__)) | ||||
| 411 | "PGO name is invalid (contains separator token)")(static_cast <bool> (StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)") ? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 411, __extension__ __PRETTY_FUNCTION__)); | ||||
| 412 | |||||
| 413 | unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); | ||||
| 414 | P += EncLen; | ||||
| 415 | |||||
| 416 | auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) { | ||||
| 417 | EncLen = encodeULEB128(CompressedLen, P); | ||||
| 418 | P += EncLen; | ||||
| 419 | char *HeaderStr = reinterpret_cast<char *>(&Header[0]); | ||||
| 420 | unsigned HeaderLen = P - &Header[0]; | ||||
| 421 | Result.append(HeaderStr, HeaderLen); | ||||
| 422 | Result += InputStr; | ||||
| 423 | return Error::success(); | ||||
| 424 | }; | ||||
| 425 | |||||
| 426 | if (!doCompression) { | ||||
| 427 | return WriteStringToResult(0, UncompressedNameStrings); | ||||
| 428 | } | ||||
| 429 | |||||
| 430 | SmallString<128> CompressedNameStrings; | ||||
| 431 | Error E = zlib::compress(StringRef(UncompressedNameStrings), | ||||
| 432 | CompressedNameStrings, zlib::BestSizeCompression); | ||||
| 433 | if (E) { | ||||
| 434 | consumeError(std::move(E)); | ||||
| 435 | return make_error<InstrProfError>(instrprof_error::compress_failed); | ||||
| 436 | } | ||||
| 437 | |||||
| 438 | return WriteStringToResult(CompressedNameStrings.size(), | ||||
| 439 | CompressedNameStrings); | ||||
| 440 | } | ||||
| 441 | |||||
| 442 | StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) { | ||||
| 443 | auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer()); | ||||
| 444 | StringRef NameStr = | ||||
| 445 | Arr->isCString() ? Arr->getAsCString() : Arr->getAsString(); | ||||
| 446 | return NameStr; | ||||
| 447 | } | ||||
| 448 | |||||
| 449 | Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, | ||||
| 450 | std::string &Result, bool doCompression) { | ||||
| 451 | std::vector<std::string> NameStrs; | ||||
| 452 | for (auto *NameVar : NameVars) { | ||||
| 453 | NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar))); | ||||
| 454 | } | ||||
| 455 | return collectPGOFuncNameStrings( | ||||
| 456 | NameStrs, zlib::isAvailable() && doCompression, Result); | ||||
| 457 | } | ||||
| 458 | |||||
| 459 | Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { | ||||
| 460 | const uint8_t *P = NameStrings.bytes_begin(); | ||||
| 461 | const uint8_t *EndP = NameStrings.bytes_end(); | ||||
| 462 | while (P < EndP) { | ||||
| |||||
| 463 | uint32_t N; | ||||
| 464 | uint64_t UncompressedSize = decodeULEB128(P, &N); | ||||
| 465 | P += N; | ||||
| 466 | uint64_t CompressedSize = decodeULEB128(P, &N); | ||||
| 467 | P += N; | ||||
| 468 | bool isCompressed = (CompressedSize != 0); | ||||
| 469 | SmallString<128> UncompressedNameStrings; | ||||
| 470 | StringRef NameStrings; | ||||
| 471 | if (isCompressed
| ||||
| 472 | if (!llvm::zlib::isAvailable()) | ||||
| 473 | return make_error<InstrProfError>(instrprof_error::zlib_unavailable); | ||||
| 474 | |||||
| 475 | StringRef CompressedNameStrings(reinterpret_cast<const char *>(P), | ||||
| 476 | CompressedSize); | ||||
| 477 | if (Error E = | ||||
| 478 | zlib::uncompress(CompressedNameStrings, UncompressedNameStrings, | ||||
| 479 | UncompressedSize)) { | ||||
| 480 | consumeError(std::move(E)); | ||||
| 481 | return make_error<InstrProfError>(instrprof_error::uncompress_failed); | ||||
| 482 | } | ||||
| 483 | P += CompressedSize; | ||||
| 484 | NameStrings = StringRef(UncompressedNameStrings.data(), | ||||
| 485 | UncompressedNameStrings.size()); | ||||
| 486 | } else { | ||||
| 487 | NameStrings = | ||||
| 488 | StringRef(reinterpret_cast<const char *>(P), UncompressedSize); | ||||
| 489 | P += UncompressedSize; | ||||
| 490 | } | ||||
| 491 | // Now parse the name strings. | ||||
| 492 | SmallVector<StringRef, 0> Names; | ||||
| 493 | NameStrings.split(Names, getInstrProfNameSeparator()); | ||||
| 494 | for (StringRef &Name : Names) | ||||
| 495 | if (Error E = Symtab.addFuncName(Name)) | ||||
| 496 | return E; | ||||
| 497 | |||||
| 498 | while (P
| ||||
| |||||
| 499 | P++; | ||||
| 500 | } | ||||
| 501 | return Error::success(); | ||||
| 502 | } | ||||
| 503 | |||||
| 504 | void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { | ||||
| 505 | uint64_t FuncSum = 0; | ||||
| 506 | Sum.NumEntries += Counts.size(); | ||||
| 507 | for (size_t F = 0, E = Counts.size(); F < E; ++F) | ||||
| 508 | FuncSum += Counts[F]; | ||||
| 509 | Sum.CountSum += FuncSum; | ||||
| 510 | |||||
| 511 | for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) { | ||||
| 512 | uint64_t KindSum = 0; | ||||
| 513 | uint32_t NumValueSites = getNumValueSites(VK); | ||||
| 514 | for (size_t I = 0; I < NumValueSites; ++I) { | ||||
| 515 | uint32_t NV = getNumValueDataForSite(VK, I); | ||||
| 516 | std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I); | ||||
| 517 | for (uint32_t V = 0; V < NV; V++) | ||||
| 518 | KindSum += VD[V].Count; | ||||
| 519 | } | ||||
| 520 | Sum.ValueCounts[VK] += KindSum; | ||||
| 521 | } | ||||
| 522 | } | ||||
| 523 | |||||
| 524 | void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input, | ||||
| 525 | uint32_t ValueKind, | ||||
| 526 | OverlapStats &Overlap, | ||||
| 527 | OverlapStats &FuncLevelOverlap) { | ||||
| 528 | this->sortByTargetValues(); | ||||
| 529 | Input.sortByTargetValues(); | ||||
| 530 | double Score = 0.0f, FuncLevelScore = 0.0f; | ||||
| 531 | auto I = ValueData.begin(); | ||||
| 532 | auto IE = ValueData.end(); | ||||
| 533 | auto J = Input.ValueData.begin(); | ||||
| 534 | auto JE = Input.ValueData.end(); | ||||
| 535 | while (I != IE && J != JE) { | ||||
| 536 | if (I->Value == J->Value) { | ||||
| 537 | Score += OverlapStats::score(I->Count, J->Count, | ||||
| 538 | Overlap.Base.ValueCounts[ValueKind], | ||||
| 539 | Overlap.Test.ValueCounts[ValueKind]); | ||||
| 540 | FuncLevelScore += OverlapStats::score( | ||||
| 541 | I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind], | ||||
| 542 | FuncLevelOverlap.Test.ValueCounts[ValueKind]); | ||||
| 543 | ++I; | ||||
| 544 | } else if (I->Value < J->Value) { | ||||
| 545 | ++I; | ||||
| 546 | continue; | ||||
| 547 | } | ||||
| 548 | ++J; | ||||
| 549 | } | ||||
| 550 | Overlap.Overlap.ValueCounts[ValueKind] += Score; | ||||
| 551 | FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore; | ||||
| 552 | } | ||||
| 553 | |||||
| 554 | // Return false on mismatch. | ||||
| 555 | void InstrProfRecord::overlapValueProfData(uint32_t ValueKind, | ||||
| 556 | InstrProfRecord &Other, | ||||
| 557 | OverlapStats &Overlap, | ||||
| 558 | OverlapStats &FuncLevelOverlap) { | ||||
| 559 | uint32_t ThisNumValueSites = getNumValueSites(ValueKind); | ||||
| 560 | assert(ThisNumValueSites == Other.getNumValueSites(ValueKind))(static_cast <bool> (ThisNumValueSites == Other.getNumValueSites (ValueKind)) ? void (0) : __assert_fail ("ThisNumValueSites == Other.getNumValueSites(ValueKind)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 560, __extension__ __PRETTY_FUNCTION__)); | ||||
| 561 | if (!ThisNumValueSites) | ||||
| 562 | return; | ||||
| 563 | |||||
| 564 | std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = | ||||
| 565 | getOrCreateValueSitesForKind(ValueKind); | ||||
| 566 | MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = | ||||
| 567 | Other.getValueSitesForKind(ValueKind); | ||||
| 568 | for (uint32_t I = 0; I < ThisNumValueSites; I++) | ||||
| 569 | ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap, | ||||
| 570 | FuncLevelOverlap); | ||||
| 571 | } | ||||
| 572 | |||||
| 573 | void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap, | ||||
| 574 | OverlapStats &FuncLevelOverlap, | ||||
| 575 | uint64_t ValueCutoff) { | ||||
| 576 | // FuncLevel CountSum for other should already computed and nonzero. | ||||
| 577 | assert(FuncLevelOverlap.Test.CountSum >= 1.0f)(static_cast <bool> (FuncLevelOverlap.Test.CountSum >= 1.0f) ? void (0) : __assert_fail ("FuncLevelOverlap.Test.CountSum >= 1.0f" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 577, __extension__ __PRETTY_FUNCTION__)); | ||||
| 578 | accumulateCounts(FuncLevelOverlap.Base); | ||||
| 579 | bool Mismatch = (Counts.size() != Other.Counts.size()); | ||||
| 580 | |||||
| 581 | // Check if the value profiles mismatch. | ||||
| 582 | if (!Mismatch) { | ||||
| 583 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { | ||||
| 584 | uint32_t ThisNumValueSites = getNumValueSites(Kind); | ||||
| 585 | uint32_t OtherNumValueSites = Other.getNumValueSites(Kind); | ||||
| 586 | if (ThisNumValueSites != OtherNumValueSites) { | ||||
| 587 | Mismatch = true; | ||||
| 588 | break; | ||||
| 589 | } | ||||
| 590 | } | ||||
| 591 | } | ||||
| 592 | if (Mismatch) { | ||||
| 593 | Overlap.addOneMismatch(FuncLevelOverlap.Test); | ||||
| 594 | return; | ||||
| 595 | } | ||||
| 596 | |||||
| 597 | // Compute overlap for value counts. | ||||
| 598 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) | ||||
| 599 | overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap); | ||||
| 600 | |||||
| 601 | double Score = 0.0; | ||||
| 602 | uint64_t MaxCount = 0; | ||||
| 603 | // Compute overlap for edge counts. | ||||
| 604 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { | ||||
| 605 | Score += OverlapStats::score(Counts[I], Other.Counts[I], | ||||
| 606 | Overlap.Base.CountSum, Overlap.Test.CountSum); | ||||
| 607 | MaxCount = std::max(Other.Counts[I], MaxCount); | ||||
| 608 | } | ||||
| 609 | Overlap.Overlap.CountSum += Score; | ||||
| 610 | Overlap.Overlap.NumEntries += 1; | ||||
| 611 | |||||
| 612 | if (MaxCount >= ValueCutoff) { | ||||
| 613 | double FuncScore = 0.0; | ||||
| 614 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) | ||||
| 615 | FuncScore += OverlapStats::score(Counts[I], Other.Counts[I], | ||||
| 616 | FuncLevelOverlap.Base.CountSum, | ||||
| 617 | FuncLevelOverlap.Test.CountSum); | ||||
| 618 | FuncLevelOverlap.Overlap.CountSum = FuncScore; | ||||
| 619 | FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size(); | ||||
| 620 | FuncLevelOverlap.Valid = true; | ||||
| 621 | } | ||||
| 622 | } | ||||
| 623 | |||||
| 624 | void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, | ||||
| 625 | uint64_t Weight, | ||||
| 626 | function_ref<void(instrprof_error)> Warn) { | ||||
| 627 | this->sortByTargetValues(); | ||||
| 628 | Input.sortByTargetValues(); | ||||
| 629 | auto I = ValueData.begin(); | ||||
| 630 | auto IE = ValueData.end(); | ||||
| 631 | for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE; | ||||
| 632 | ++J) { | ||||
| 633 | while (I != IE && I->Value < J->Value) | ||||
| 634 | ++I; | ||||
| 635 | if (I != IE && I->Value == J->Value) { | ||||
| 636 | bool Overflowed; | ||||
| 637 | I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed); | ||||
| 638 | if (Overflowed) | ||||
| 639 | Warn(instrprof_error::counter_overflow); | ||||
| 640 | ++I; | ||||
| 641 | continue; | ||||
| 642 | } | ||||
| 643 | ValueData.insert(I, *J); | ||||
| 644 | } | ||||
| 645 | } | ||||
| 646 | |||||
| 647 | void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D, | ||||
| 648 | function_ref<void(instrprof_error)> Warn) { | ||||
| 649 | for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) { | ||||
| 650 | bool Overflowed; | ||||
| 651 | I->Count = SaturatingMultiply(I->Count, N, &Overflowed) / D; | ||||
| 652 | if (Overflowed) | ||||
| 653 | Warn(instrprof_error::counter_overflow); | ||||
| 654 | } | ||||
| 655 | } | ||||
| 656 | |||||
| 657 | // Merge Value Profile data from Src record to this record for ValueKind. | ||||
| 658 | // Scale merged value counts by \p Weight. | ||||
| 659 | void InstrProfRecord::mergeValueProfData( | ||||
| 660 | uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight, | ||||
| 661 | function_ref<void(instrprof_error)> Warn) { | ||||
| 662 | uint32_t ThisNumValueSites = getNumValueSites(ValueKind); | ||||
| 663 | uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); | ||||
| 664 | if (ThisNumValueSites != OtherNumValueSites) { | ||||
| 665 | Warn(instrprof_error::value_site_count_mismatch); | ||||
| 666 | return; | ||||
| 667 | } | ||||
| 668 | if (!ThisNumValueSites) | ||||
| 669 | return; | ||||
| 670 | std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = | ||||
| 671 | getOrCreateValueSitesForKind(ValueKind); | ||||
| 672 | MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = | ||||
| 673 | Src.getValueSitesForKind(ValueKind); | ||||
| 674 | for (uint32_t I = 0; I < ThisNumValueSites; I++) | ||||
| 675 | ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn); | ||||
| 676 | } | ||||
| 677 | |||||
| 678 | void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, | ||||
| 679 | function_ref<void(instrprof_error)> Warn) { | ||||
| 680 | // If the number of counters doesn't match we either have bad data | ||||
| 681 | // or a hash collision. | ||||
| 682 | if (Counts.size() != Other.Counts.size()) { | ||||
| 683 | Warn(instrprof_error::count_mismatch); | ||||
| 684 | return; | ||||
| 685 | } | ||||
| 686 | |||||
| 687 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { | ||||
| 688 | bool Overflowed; | ||||
| 689 | Counts[I] = | ||||
| 690 | SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); | ||||
| 691 | if (Overflowed) | ||||
| 692 | Warn(instrprof_error::counter_overflow); | ||||
| 693 | } | ||||
| 694 | |||||
| 695 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) | ||||
| 696 | mergeValueProfData(Kind, Other, Weight, Warn); | ||||
| 697 | } | ||||
| 698 | |||||
| 699 | void InstrProfRecord::scaleValueProfData( | ||||
| 700 | uint32_t ValueKind, uint64_t N, uint64_t D, | ||||
| 701 | function_ref<void(instrprof_error)> Warn) { | ||||
| 702 | for (auto &R : getValueSitesForKind(ValueKind)) | ||||
| 703 | R.scale(N, D, Warn); | ||||
| 704 | } | ||||
| 705 | |||||
| 706 | void InstrProfRecord::scale(uint64_t N, uint64_t D, | ||||
| 707 | function_ref<void(instrprof_error)> Warn) { | ||||
| 708 | assert(D != 0 && "D cannot be 0")(static_cast <bool> (D != 0 && "D cannot be 0") ? void (0) : __assert_fail ("D != 0 && \"D cannot be 0\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 708, __extension__ __PRETTY_FUNCTION__)); | ||||
| 709 | for (auto &Count : this->Counts) { | ||||
| 710 | bool Overflowed; | ||||
| 711 | Count = SaturatingMultiply(Count, N, &Overflowed) / D; | ||||
| 712 | if (Overflowed) | ||||
| 713 | Warn(instrprof_error::counter_overflow); | ||||
| 714 | } | ||||
| 715 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) | ||||
| 716 | scaleValueProfData(Kind, N, D, Warn); | ||||
| 717 | } | ||||
| 718 | |||||
| 719 | // Map indirect call target name hash to name string. | ||||
| 720 | uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, | ||||
| 721 | InstrProfSymtab *SymTab) { | ||||
| 722 | if (!SymTab) | ||||
| 723 | return Value; | ||||
| 724 | |||||
| 725 | if (ValueKind == IPVK_IndirectCallTarget) | ||||
| 726 | return SymTab->getFunctionHashFromAddress(Value); | ||||
| 727 | |||||
| 728 | return Value; | ||||
| 729 | } | ||||
| 730 | |||||
| 731 | void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, | ||||
| 732 | InstrProfValueData *VData, uint32_t N, | ||||
| 733 | InstrProfSymtab *ValueMap) { | ||||
| 734 | for (uint32_t I = 0; I < N; I++) { | ||||
| 735 | VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap); | ||||
| 736 | } | ||||
| 737 | std::vector<InstrProfValueSiteRecord> &ValueSites = | ||||
| 738 | getOrCreateValueSitesForKind(ValueKind); | ||||
| 739 | if (N == 0) | ||||
| 740 | ValueSites.emplace_back(); | ||||
| 741 | else | ||||
| 742 | ValueSites.emplace_back(VData, VData + N); | ||||
| 743 | } | ||||
| 744 | |||||
| 745 | #define INSTR_PROF_COMMON_API_IMPL | ||||
| 746 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
| 747 | |||||
| 748 | /*! | ||||
| 749 | * ValueProfRecordClosure Interface implementation for InstrProfRecord | ||||
| 750 | * class. These C wrappers are used as adaptors so that C++ code can be | ||||
| 751 | * invoked as callbacks. | ||||
| 752 | */ | ||||
| 753 | uint32_t getNumValueKindsInstrProf(const void *Record) { | ||||
| 754 | return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds(); | ||||
| 755 | } | ||||
| 756 | |||||
| 757 | uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) { | ||||
| 758 | return reinterpret_cast<const InstrProfRecord *>(Record) | ||||
| 759 | ->getNumValueSites(VKind); | ||||
| 760 | } | ||||
| 761 | |||||
| 762 | uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) { | ||||
| 763 | return reinterpret_cast<const InstrProfRecord *>(Record) | ||||
| 764 | ->getNumValueData(VKind); | ||||
| 765 | } | ||||
| 766 | |||||
| 767 | uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, | ||||
| 768 | uint32_t S) { | ||||
| 769 | return reinterpret_cast<const InstrProfRecord *>(R) | ||||
| 770 | ->getNumValueDataForSite(VK, S); | ||||
| 771 | } | ||||
| 772 | |||||
| 773 | void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, | ||||
| 774 | uint32_t K, uint32_t S) { | ||||
| 775 | reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S); | ||||
| 776 | } | ||||
| 777 | |||||
| 778 | ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { | ||||
| 779 | ValueProfData *VD = | ||||
| 780 | (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData()); | ||||
| 781 | memset(VD, 0, TotalSizeInBytes); | ||||
| 782 | return VD; | ||||
| 783 | } | ||||
| 784 | |||||
| 785 | static ValueProfRecordClosure InstrProfRecordClosure = { | ||||
| 786 | nullptr, | ||||
| 787 | getNumValueKindsInstrProf, | ||||
| 788 | getNumValueSitesInstrProf, | ||||
| 789 | getNumValueDataInstrProf, | ||||
| 790 | getNumValueDataForSiteInstrProf, | ||||
| 791 | nullptr, | ||||
| 792 | getValueForSiteInstrProf, | ||||
| 793 | allocValueProfDataInstrProf}; | ||||
| 794 | |||||
| 795 | // Wrapper implementation using the closure mechanism. | ||||
| 796 | uint32_t ValueProfData::getSize(const InstrProfRecord &Record) { | ||||
| 797 | auto Closure = InstrProfRecordClosure; | ||||
| 798 | Closure.Record = &Record; | ||||
| 799 | return getValueProfDataSize(&Closure); | ||||
| 800 | } | ||||
| 801 | |||||
| 802 | // Wrapper implementation using the closure mechanism. | ||||
| 803 | std::unique_ptr<ValueProfData> | ||||
| 804 | ValueProfData::serializeFrom(const InstrProfRecord &Record) { | ||||
| 805 | InstrProfRecordClosure.Record = &Record; | ||||
| 806 | |||||
| 807 | std::unique_ptr<ValueProfData> VPD( | ||||
| 808 | serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr)); | ||||
| 809 | return VPD; | ||||
| 810 | } | ||||
| 811 | |||||
| 812 | void ValueProfRecord::deserializeTo(InstrProfRecord &Record, | ||||
| 813 | InstrProfSymtab *SymTab) { | ||||
| 814 | Record.reserveSites(Kind, NumValueSites); | ||||
| 815 | |||||
| 816 | InstrProfValueData *ValueData = getValueProfRecordValueData(this); | ||||
| 817 | for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) { | ||||
| 818 | uint8_t ValueDataCount = this->SiteCountArray[VSite]; | ||||
| 819 | Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab); | ||||
| 820 | ValueData += ValueDataCount; | ||||
| 821 | } | ||||
| 822 | } | ||||
| 823 | |||||
| 824 | // For writing/serializing, Old is the host endianness, and New is | ||||
| 825 | // byte order intended on disk. For Reading/deserialization, Old | ||||
| 826 | // is the on-disk source endianness, and New is the host endianness. | ||||
| 827 | void ValueProfRecord::swapBytes(support::endianness Old, | ||||
| 828 | support::endianness New) { | ||||
| 829 | using namespace support; | ||||
| 830 | |||||
| 831 | if (Old == New) | ||||
| 832 | return; | ||||
| 833 | |||||
| 834 | if (getHostEndianness() != Old) { | ||||
| 835 | sys::swapByteOrder<uint32_t>(NumValueSites); | ||||
| 836 | sys::swapByteOrder<uint32_t>(Kind); | ||||
| 837 | } | ||||
| 838 | uint32_t ND = getValueProfRecordNumValueData(this); | ||||
| 839 | InstrProfValueData *VD = getValueProfRecordValueData(this); | ||||
| 840 | |||||
| 841 | // No need to swap byte array: SiteCountArrray. | ||||
| 842 | for (uint32_t I = 0; I < ND; I++) { | ||||
| 843 | sys::swapByteOrder<uint64_t>(VD[I].Value); | ||||
| 844 | sys::swapByteOrder<uint64_t>(VD[I].Count); | ||||
| 845 | } | ||||
| 846 | if (getHostEndianness() == Old) { | ||||
| 847 | sys::swapByteOrder<uint32_t>(NumValueSites); | ||||
| 848 | sys::swapByteOrder<uint32_t>(Kind); | ||||
| 849 | } | ||||
| 850 | } | ||||
| 851 | |||||
| 852 | void ValueProfData::deserializeTo(InstrProfRecord &Record, | ||||
| 853 | InstrProfSymtab *SymTab) { | ||||
| 854 | if (NumValueKinds == 0) | ||||
| 855 | return; | ||||
| 856 | |||||
| 857 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
| 858 | for (uint32_t K = 0; K < NumValueKinds; K++) { | ||||
| 859 | VR->deserializeTo(Record, SymTab); | ||||
| 860 | VR = getValueProfRecordNext(VR); | ||||
| 861 | } | ||||
| 862 | } | ||||
| 863 | |||||
| 864 | template <class T> | ||||
| 865 | static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) { | ||||
| 866 | using namespace support; | ||||
| 867 | |||||
| 868 | if (Orig == little) | ||||
| 869 | return endian::readNext<T, little, unaligned>(D); | ||||
| 870 | else | ||||
| 871 | return endian::readNext<T, big, unaligned>(D); | ||||
| 872 | } | ||||
| 873 | |||||
| 874 | static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) { | ||||
| 875 | return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize)) | ||||
| 876 | ValueProfData()); | ||||
| 877 | } | ||||
| 878 | |||||
| 879 | Error ValueProfData::checkIntegrity() { | ||||
| 880 | if (NumValueKinds > IPVK_Last + 1) | ||||
| 881 | return make_error<InstrProfError>(instrprof_error::malformed); | ||||
| 882 | // Total size needs to be mulltiple of quadword size. | ||||
| 883 | if (TotalSize % sizeof(uint64_t)) | ||||
| 884 | return make_error<InstrProfError>(instrprof_error::malformed); | ||||
| 885 | |||||
| 886 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
| 887 | for (uint32_t K = 0; K < this->NumValueKinds; K++) { | ||||
| 888 | if (VR->Kind > IPVK_Last) | ||||
| 889 | return make_error<InstrProfError>(instrprof_error::malformed); | ||||
| 890 | VR = getValueProfRecordNext(VR); | ||||
| 891 | if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize) | ||||
| 892 | return make_error<InstrProfError>(instrprof_error::malformed); | ||||
| 893 | } | ||||
| 894 | return Error::success(); | ||||
| 895 | } | ||||
| 896 | |||||
| 897 | Expected<std::unique_ptr<ValueProfData>> | ||||
| 898 | ValueProfData::getValueProfData(const unsigned char *D, | ||||
| 899 | const unsigned char *const BufferEnd, | ||||
| 900 | support::endianness Endianness) { | ||||
| 901 | using namespace support; | ||||
| 902 | |||||
| 903 | if (D + sizeof(ValueProfData) > BufferEnd) | ||||
| 904 | return make_error<InstrProfError>(instrprof_error::truncated); | ||||
| 905 | |||||
| 906 | const unsigned char *Header = D; | ||||
| 907 | uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness); | ||||
| 908 | if (D + TotalSize > BufferEnd) | ||||
| 909 | return make_error<InstrProfError>(instrprof_error::too_large); | ||||
| 910 | |||||
| 911 | std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize); | ||||
| 912 | memcpy(VPD.get(), D, TotalSize); | ||||
| 913 | // Byte swap. | ||||
| 914 | VPD->swapBytesToHost(Endianness); | ||||
| 915 | |||||
| 916 | Error E = VPD->checkIntegrity(); | ||||
| 917 | if (E) | ||||
| 918 | return std::move(E); | ||||
| 919 | |||||
| 920 | return std::move(VPD); | ||||
| 921 | } | ||||
| 922 | |||||
| 923 | void ValueProfData::swapBytesToHost(support::endianness Endianness) { | ||||
| 924 | using namespace support; | ||||
| 925 | |||||
| 926 | if (Endianness == getHostEndianness()) | ||||
| 927 | return; | ||||
| 928 | |||||
| 929 | sys::swapByteOrder<uint32_t>(TotalSize); | ||||
| 930 | sys::swapByteOrder<uint32_t>(NumValueKinds); | ||||
| 931 | |||||
| 932 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
| 933 | for (uint32_t K = 0; K < NumValueKinds; K++) { | ||||
| 934 | VR->swapBytes(Endianness, getHostEndianness()); | ||||
| 935 | VR = getValueProfRecordNext(VR); | ||||
| 936 | } | ||||
| 937 | } | ||||
| 938 | |||||
| 939 | void ValueProfData::swapBytesFromHost(support::endianness Endianness) { | ||||
| 940 | using namespace support; | ||||
| 941 | |||||
| 942 | if (Endianness == getHostEndianness()) | ||||
| 943 | return; | ||||
| 944 | |||||
| 945 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
| 946 | for (uint32_t K = 0; K < NumValueKinds; K++) { | ||||
| 947 | ValueProfRecord *NVR = getValueProfRecordNext(VR); | ||||
| 948 | VR->swapBytes(getHostEndianness(), Endianness); | ||||
| 949 | VR = NVR; | ||||
| 950 | } | ||||
| 951 | sys::swapByteOrder<uint32_t>(TotalSize); | ||||
| 952 | sys::swapByteOrder<uint32_t>(NumValueKinds); | ||||
| 953 | } | ||||
| 954 | |||||
| 955 | void annotateValueSite(Module &M, Instruction &Inst, | ||||
| 956 | const InstrProfRecord &InstrProfR, | ||||
| 957 | InstrProfValueKind ValueKind, uint32_t SiteIdx, | ||||
| 958 | uint32_t MaxMDCount) { | ||||
| 959 | uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx); | ||||
| 960 | if (!NV) | ||||
| 961 | return; | ||||
| 962 | |||||
| 963 | uint64_t Sum = 0; | ||||
| 964 | std::unique_ptr<InstrProfValueData[]> VD = | ||||
| 965 | InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum); | ||||
| 966 | |||||
| 967 | ArrayRef<InstrProfValueData> VDs(VD.get(), NV); | ||||
| 968 | annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount); | ||||
| 969 | } | ||||
| 970 | |||||
| 971 | void annotateValueSite(Module &M, Instruction &Inst, | ||||
| 972 | ArrayRef<InstrProfValueData> VDs, | ||||
| 973 | uint64_t Sum, InstrProfValueKind ValueKind, | ||||
| 974 | uint32_t MaxMDCount) { | ||||
| 975 | LLVMContext &Ctx = M.getContext(); | ||||
| 976 | MDBuilder MDHelper(Ctx); | ||||
| 977 | SmallVector<Metadata *, 3> Vals; | ||||
| 978 | // Tag | ||||
| 979 | Vals.push_back(MDHelper.createString("VP")); | ||||
| 980 | // Value Kind | ||||
| 981 | Vals.push_back(MDHelper.createConstant( | ||||
| 982 | ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind))); | ||||
| 983 | // Total Count | ||||
| 984 | Vals.push_back( | ||||
| 985 | MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum))); | ||||
| 986 | |||||
| 987 | // Value Profile Data | ||||
| 988 | uint32_t MDCount = MaxMDCount; | ||||
| 989 | for (auto &VD : VDs) { | ||||
| 990 | Vals.push_back(MDHelper.createConstant( | ||||
| 991 | ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value))); | ||||
| 992 | Vals.push_back(MDHelper.createConstant( | ||||
| 993 | ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count))); | ||||
| 994 | if (--MDCount == 0) | ||||
| 995 | break; | ||||
| 996 | } | ||||
| 997 | Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); | ||||
| 998 | } | ||||
| 999 | |||||
| 1000 | bool getValueProfDataFromInst(const Instruction &Inst, | ||||
| 1001 | InstrProfValueKind ValueKind, | ||||
| 1002 | uint32_t MaxNumValueData, | ||||
| 1003 | InstrProfValueData ValueData[], | ||||
| 1004 | uint32_t &ActualNumValueData, uint64_t &TotalC, | ||||
| 1005 | bool GetNoICPValue) { | ||||
| 1006 | MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); | ||||
| 1007 | if (!MD) | ||||
| 1008 | return false; | ||||
| 1009 | |||||
| 1010 | unsigned NOps = MD->getNumOperands(); | ||||
| 1011 | |||||
| 1012 | if (NOps < 5) | ||||
| 1013 | return false; | ||||
| 1014 | |||||
| 1015 | // Operand 0 is a string tag "VP": | ||||
| 1016 | MDString *Tag = cast<MDString>(MD->getOperand(0)); | ||||
| 1017 | if (!Tag) | ||||
| 1018 | return false; | ||||
| 1019 | |||||
| 1020 | if (!Tag->getString().equals("VP")) | ||||
| 1021 | return false; | ||||
| 1022 | |||||
| 1023 | // Now check kind: | ||||
| 1024 | ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1)); | ||||
| 1025 | if (!KindInt) | ||||
| 1026 | return false; | ||||
| 1027 | if (KindInt->getZExtValue() != ValueKind) | ||||
| 1028 | return false; | ||||
| 1029 | |||||
| 1030 | // Get total count | ||||
| 1031 | ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); | ||||
| 1032 | if (!TotalCInt) | ||||
| 1033 | return false; | ||||
| 1034 | TotalC = TotalCInt->getZExtValue(); | ||||
| 1035 | |||||
| 1036 | ActualNumValueData = 0; | ||||
| 1037 | |||||
| 1038 | for (unsigned I = 3; I < NOps; I += 2) { | ||||
| 1039 | if (ActualNumValueData >= MaxNumValueData) | ||||
| 1040 | break; | ||||
| 1041 | ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I)); | ||||
| 1042 | ConstantInt *Count = | ||||
| 1043 | mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1)); | ||||
| 1044 | if (!Value || !Count) | ||||
| 1045 | return false; | ||||
| 1046 | uint64_t CntValue = Count->getZExtValue(); | ||||
| 1047 | if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM)) | ||||
| 1048 | continue; | ||||
| 1049 | ValueData[ActualNumValueData].Value = Value->getZExtValue(); | ||||
| 1050 | ValueData[ActualNumValueData].Count = CntValue; | ||||
| 1051 | ActualNumValueData++; | ||||
| 1052 | } | ||||
| 1053 | return true; | ||||
| 1054 | } | ||||
| 1055 | |||||
| 1056 | MDNode *getPGOFuncNameMetadata(const Function &F) { | ||||
| 1057 | return F.getMetadata(getPGOFuncNameMetadataName()); | ||||
| 1058 | } | ||||
| 1059 | |||||
| 1060 | void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { | ||||
| 1061 | // Only for internal linkage functions. | ||||
| 1062 | if (PGOFuncName == F.getName()) | ||||
| 1063 | return; | ||||
| 1064 | // Don't create duplicated meta-data. | ||||
| 1065 | if (getPGOFuncNameMetadata(F)) | ||||
| 1066 | return; | ||||
| 1067 | LLVMContext &C = F.getContext(); | ||||
| 1068 | MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName)); | ||||
| 1069 | F.setMetadata(getPGOFuncNameMetadataName(), N); | ||||
| 1070 | } | ||||
| 1071 | |||||
| 1072 | bool needsComdatForCounter(const Function &F, const Module &M) { | ||||
| 1073 | if (F.hasComdat()) | ||||
| 1074 | return true; | ||||
| 1075 | |||||
| 1076 | if (!Triple(M.getTargetTriple()).supportsCOMDAT()) | ||||
| 1077 | return false; | ||||
| 1078 | |||||
| 1079 | // See createPGOFuncNameVar for more details. To avoid link errors, profile | ||||
| 1080 | // counters for function with available_externally linkage needs to be changed | ||||
| 1081 | // to linkonce linkage. On ELF based systems, this leads to weak symbols to be | ||||
| 1082 | // created. Without using comdat, duplicate entries won't be removed by the | ||||
| 1083 | // linker leading to increased data segement size and raw profile size. Even | ||||
| 1084 | // worse, since the referenced counter from profile per-function data object | ||||
| 1085 | // will be resolved to the common strong definition, the profile counts for | ||||
| 1086 | // available_externally functions will end up being duplicated in raw profile | ||||
| 1087 | // data. This can result in distorted profile as the counts of those dups | ||||
| 1088 | // will be accumulated by the profile merger. | ||||
| 1089 | GlobalValue::LinkageTypes Linkage = F.getLinkage(); | ||||
| 1090 | if (Linkage != GlobalValue::ExternalWeakLinkage && | ||||
| 1091 | Linkage != GlobalValue::AvailableExternallyLinkage) | ||||
| 1092 | return false; | ||||
| 1093 | |||||
| 1094 | return true; | ||||
| 1095 | } | ||||
| 1096 | |||||
| 1097 | // Check if INSTR_PROF_RAW_VERSION_VAR is defined. | ||||
| 1098 | bool isIRPGOFlagSet(const Module *M) { | ||||
| 1099 | auto IRInstrVar = | ||||
| 1100 | M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version"); | ||||
| 1101 | if (!IRInstrVar || IRInstrVar->hasLocalLinkage()) | ||||
| 1102 | return false; | ||||
| 1103 | |||||
| 1104 | // For CSPGO+LTO, this variable might be marked as non-prevailing and we only | ||||
| 1105 | // have the decl. | ||||
| 1106 | if (IRInstrVar->isDeclaration()) | ||||
| 1107 | return true; | ||||
| 1108 | |||||
| 1109 | // Check if the flag is set. | ||||
| 1110 | if (!IRInstrVar->hasInitializer()) | ||||
| 1111 | return false; | ||||
| 1112 | |||||
| 1113 | auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer()); | ||||
| 1114 | if (!InitVal) | ||||
| 1115 | return false; | ||||
| 1116 | return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF(0x1ULL << 56)) != 0; | ||||
| 1117 | } | ||||
| 1118 | |||||
| 1119 | // Check if we can safely rename this Comdat function. | ||||
| 1120 | bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) { | ||||
| 1121 | if (F.getName().empty()) | ||||
| 1122 | return false; | ||||
| 1123 | if (!needsComdatForCounter(F, *(F.getParent()))) | ||||
| 1124 | return false; | ||||
| 1125 | // Unsafe to rename the address-taken function (which can be used in | ||||
| 1126 | // function comparison). | ||||
| 1127 | if (CheckAddressTaken && F.hasAddressTaken()) | ||||
| 1128 | return false; | ||||
| 1129 | // Only safe to do if this function may be discarded if it is not used | ||||
| 1130 | // in the compilation unit. | ||||
| 1131 | if (!GlobalValue::isDiscardableIfUnused(F.getLinkage())) | ||||
| 1132 | return false; | ||||
| 1133 | |||||
| 1134 | // For AvailableExternallyLinkage functions. | ||||
| 1135 | if (!F.hasComdat()) { | ||||
| 1136 | assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage)(static_cast <bool> (F.getLinkage() == GlobalValue::AvailableExternallyLinkage ) ? void (0) : __assert_fail ("F.getLinkage() == GlobalValue::AvailableExternallyLinkage" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/ProfileData/InstrProf.cpp" , 1136, __extension__ __PRETTY_FUNCTION__)); | ||||
| 1137 | return true; | ||||
| 1138 | } | ||||
| 1139 | return true; | ||||
| 1140 | } | ||||
| 1141 | |||||
| 1142 | // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime | ||||
| 1143 | // aware this is an ir_level profile so it can set the version flag. | ||||
| 1144 | GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS, | ||||
| 1145 | bool InstrEntryBBEnabled) { | ||||
| 1146 | const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version"); | ||||
| 1147 | Type *IntTy64 = Type::getInt64Ty(M.getContext()); | ||||
| 1148 | uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION7 | VARIANT_MASK_IR_PROF(0x1ULL << 56)); | ||||
| 1149 | if (IsCS) | ||||
| 1150 | ProfileVersion |= VARIANT_MASK_CSIR_PROF(0x1ULL << 57); | ||||
| 1151 | if (InstrEntryBBEnabled) | ||||
| 1152 | ProfileVersion |= VARIANT_MASK_INSTR_ENTRY(0x1ULL << 58); | ||||
| 1153 | auto IRLevelVersionVariable = new GlobalVariable( | ||||
| 1154 | M, IntTy64, true, GlobalValue::WeakAnyLinkage, | ||||
| 1155 | Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName); | ||||
| 1156 | IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility); | ||||
| 1157 | Triple TT(M.getTargetTriple()); | ||||
| 1158 | if (TT.supportsCOMDAT()) { | ||||
| 1159 | IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage); | ||||
| 1160 | IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName)); | ||||
| 1161 | } | ||||
| 1162 | return IRLevelVersionVariable; | ||||
| 1163 | } | ||||
| 1164 | |||||
| 1165 | // Create the variable for the profile file name. | ||||
| 1166 | void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) { | ||||
| 1167 | if (InstrProfileOutput.empty()) | ||||
| 1168 | return; | ||||
| 1169 | Constant *ProfileNameConst = | ||||
| 1170 | ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true); | ||||
| 1171 | GlobalVariable *ProfileNameVar = new GlobalVariable( | ||||
| 1172 | M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage, | ||||
| 1173 | ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename"); | ||||
| 1174 | Triple TT(M.getTargetTriple()); | ||||
| 1175 | if (TT.supportsCOMDAT()) { | ||||
| 1176 | ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage); | ||||
| 1177 | ProfileNameVar->setComdat(M.getOrInsertComdat( | ||||
| 1178 | StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename"))); | ||||
| 1179 | } | ||||
| 1180 | } | ||||
| 1181 | |||||
| 1182 | Error OverlapStats::accumulateCounts(const std::string &BaseFilename, | ||||
| 1183 | const std::string &TestFilename, | ||||
| 1184 | bool IsCS) { | ||||
| 1185 | auto getProfileSum = [IsCS](const std::string &Filename, | ||||
| 1186 | CountSumOrPercent &Sum) -> Error { | ||||
| 1187 | auto ReaderOrErr = InstrProfReader::create(Filename); | ||||
| 1188 | if (Error E = ReaderOrErr.takeError()) { | ||||
| 1189 | return E; | ||||
| 1190 | } | ||||
| 1191 | auto Reader = std::move(ReaderOrErr.get()); | ||||
| 1192 | Reader->accumulateCounts(Sum, IsCS); | ||||
| 1193 | return Error::success(); | ||||
| 1194 | }; | ||||
| 1195 | auto Ret = getProfileSum(BaseFilename, Base); | ||||
| 1196 | if (Ret) | ||||
| 1197 | return Ret; | ||||
| 1198 | Ret = getProfileSum(TestFilename, Test); | ||||
| 1199 | if (Ret) | ||||
| 1200 | return Ret; | ||||
| 1201 | this->BaseFilename = &BaseFilename; | ||||
| 1202 | this->TestFilename = &TestFilename; | ||||
| 1203 | Valid = true; | ||||
| 1204 | return Error::success(); | ||||
| 1205 | } | ||||
| 1206 | |||||
| 1207 | void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) { | ||||
| 1208 | Mismatch.NumEntries += 1; | ||||
| 1209 | Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum; | ||||
| 1210 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { | ||||
| 1211 | if (Test.ValueCounts[I] >= 1.0f) | ||||
| 1212 | Mismatch.ValueCounts[I] += | ||||
| 1213 | MismatchFunc.ValueCounts[I] / Test.ValueCounts[I]; | ||||
| 1214 | } | ||||
| 1215 | } | ||||
| 1216 | |||||
| 1217 | void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) { | ||||
| 1218 | Unique.NumEntries += 1; | ||||
| 1219 | Unique.CountSum += UniqueFunc.CountSum / Test.CountSum; | ||||
| 1220 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { | ||||
| 1221 | if (Test.ValueCounts[I] >= 1.0f) | ||||
| 1222 | Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I]; | ||||
| 1223 | } | ||||
| 1224 | } | ||||
| 1225 | |||||
| 1226 | void OverlapStats::dump(raw_fd_ostream &OS) const { | ||||
| 1227 | if (!Valid) | ||||
| 1228 | return; | ||||
| 1229 | |||||
| 1230 | const char *EntryName = | ||||
| 1231 | (Level == ProgramLevel ? "functions" : "edge counters"); | ||||
| 1232 | if (Level == ProgramLevel) { | ||||
| 1233 | OS << "Profile overlap infomation for base_profile: " << *BaseFilename | ||||
| 1234 | << " and test_profile: " << *TestFilename << "\nProgram level:\n"; | ||||
| 1235 | } else { | ||||
| 1236 | OS << "Function level:\n" | ||||
| 1237 | << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n"; | ||||
| 1238 | } | ||||
| 1239 | |||||
| 1240 | OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n"; | ||||
| 1241 | if (Mismatch.NumEntries) | ||||
| 1242 | OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries | ||||
| 1243 | << "\n"; | ||||
| 1244 | if (Unique.NumEntries) | ||||
| 1245 | OS << " # of " << EntryName | ||||
| 1246 | << " only in test_profile: " << Unique.NumEntries << "\n"; | ||||
| 1247 | |||||
| 1248 | OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100) | ||||
| 1249 | << "\n"; | ||||
| 1250 | if (Mismatch.NumEntries) | ||||
| 1251 | OS << " Mismatched count percentage (Edge): " | ||||
| 1252 | << format("%.3f%%", Mismatch.CountSum * 100) << "\n"; | ||||
| 1253 | if (Unique.NumEntries) | ||||
| 1254 | OS << " Percentage of Edge profile only in test_profile: " | ||||
| 1255 | << format("%.3f%%", Unique.CountSum * 100) << "\n"; | ||||
| 1256 | OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum) | ||||
| 1257 | << "\n" | ||||
| 1258 | << " Edge profile test count sum: " << format("%.0f", Test.CountSum) | ||||
| 1259 | << "\n"; | ||||
| 1260 | |||||
| 1261 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { | ||||
| 1262 | if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) | ||||
| 1263 | continue; | ||||
| 1264 | char ProfileKindName[20]; | ||||
| 1265 | switch (I) { | ||||
| 1266 | case IPVK_IndirectCallTarget: | ||||
| 1267 | strncpy(ProfileKindName, "IndirectCall", 19); | ||||
| 1268 | break; | ||||
| 1269 | case IPVK_MemOPSize: | ||||
| 1270 | strncpy(ProfileKindName, "MemOP", 19); | ||||
| 1271 | break; | ||||
| 1272 | default: | ||||
| 1273 | snprintf(ProfileKindName, 19, "VP[%d]", I); | ||||
| 1274 | break; | ||||
| 1275 | } | ||||
| 1276 | OS << " " << ProfileKindName | ||||
| 1277 | << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100) | ||||
| 1278 | << "\n"; | ||||
| 1279 | if (Mismatch.NumEntries) | ||||
| 1280 | OS << " Mismatched count percentage (" << ProfileKindName | ||||
| 1281 | << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n"; | ||||
| 1282 | if (Unique.NumEntries) | ||||
| 1283 | OS << " Percentage of " << ProfileKindName | ||||
| 1284 | << " profile only in test_profile: " | ||||
| 1285 | << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n"; | ||||
| 1286 | OS << " " << ProfileKindName | ||||
| 1287 | << " profile base count sum: " << format("%.0f", Base.ValueCounts[I]) | ||||
| 1288 | << "\n" | ||||
| 1289 | << " " << ProfileKindName | ||||
| 1290 | << " profile test count sum: " << format("%.0f", Test.ValueCounts[I]) | ||||
| 1291 | << "\n"; | ||||
| 1292 | } | ||||
| 1293 | } | ||||
| 1294 | |||||
| 1295 | } // end namespace llvm |
| 1 | //===- llvm/Support/LEB128.h - [SU]LEB128 utility functions -----*- C++ -*-===// | ||||
| 2 | // | ||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
| 6 | // | ||||
| 7 | //===----------------------------------------------------------------------===// | ||||
| 8 | // | ||||
| 9 | // This file declares some utility functions for encoding SLEB128 and | ||||
| 10 | // ULEB128 values. | ||||
| 11 | // | ||||
| 12 | //===----------------------------------------------------------------------===// | ||||
| 13 | |||||
| 14 | #ifndef LLVM_SUPPORT_LEB128_H | ||||
| 15 | #define LLVM_SUPPORT_LEB128_H | ||||
| 16 | |||||
| 17 | #include "llvm/Support/raw_ostream.h" | ||||
| 18 | |||||
| 19 | namespace llvm { | ||||
| 20 | |||||
| 21 | /// Utility function to encode a SLEB128 value to an output stream. Returns | ||||
| 22 | /// the length in bytes of the encoded value. | ||||
| 23 | inline unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, | ||||
| 24 | unsigned PadTo = 0) { | ||||
| 25 | bool More; | ||||
| 26 | unsigned Count = 0; | ||||
| 27 | do { | ||||
| 28 | uint8_t Byte = Value & 0x7f; | ||||
| 29 | // NOTE: this assumes that this signed shift is an arithmetic right shift. | ||||
| 30 | Value >>= 7; | ||||
| 31 | More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || | ||||
| 32 | ((Value == -1) && ((Byte & 0x40) != 0)))); | ||||
| 33 | Count++; | ||||
| 34 | if (More || Count < PadTo) | ||||
| 35 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
| 36 | OS << char(Byte); | ||||
| 37 | } while (More); | ||||
| 38 | |||||
| 39 | // Pad with 0x80 and emit a terminating byte at the end. | ||||
| 40 | if (Count < PadTo) { | ||||
| 41 | uint8_t PadValue = Value < 0 ? 0x7f : 0x00; | ||||
| 42 | for (; Count < PadTo - 1; ++Count) | ||||
| 43 | OS << char(PadValue | 0x80); | ||||
| 44 | OS << char(PadValue); | ||||
| 45 | Count++; | ||||
| 46 | } | ||||
| 47 | return Count; | ||||
| 48 | } | ||||
| 49 | |||||
| 50 | /// Utility function to encode a SLEB128 value to a buffer. Returns | ||||
| 51 | /// the length in bytes of the encoded value. | ||||
| 52 | inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned PadTo = 0) { | ||||
| 53 | uint8_t *orig_p = p; | ||||
| 54 | unsigned Count = 0; | ||||
| 55 | bool More; | ||||
| 56 | do { | ||||
| 57 | uint8_t Byte = Value & 0x7f; | ||||
| 58 | // NOTE: this assumes that this signed shift is an arithmetic right shift. | ||||
| 59 | Value >>= 7; | ||||
| 60 | More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || | ||||
| 61 | ((Value == -1) && ((Byte & 0x40) != 0)))); | ||||
| 62 | Count++; | ||||
| 63 | if (More || Count < PadTo) | ||||
| 64 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
| 65 | *p++ = Byte; | ||||
| 66 | } while (More); | ||||
| 67 | |||||
| 68 | // Pad with 0x80 and emit a terminating byte at the end. | ||||
| 69 | if (Count < PadTo) { | ||||
| 70 | uint8_t PadValue = Value < 0 ? 0x7f : 0x00; | ||||
| 71 | for (; Count < PadTo - 1; ++Count) | ||||
| 72 | *p++ = (PadValue | 0x80); | ||||
| 73 | *p++ = PadValue; | ||||
| 74 | } | ||||
| 75 | return (unsigned)(p - orig_p); | ||||
| 76 | } | ||||
| 77 | |||||
| 78 | /// Utility function to encode a ULEB128 value to an output stream. Returns | ||||
| 79 | /// the length in bytes of the encoded value. | ||||
| 80 | inline unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, | ||||
| 81 | unsigned PadTo = 0) { | ||||
| 82 | unsigned Count = 0; | ||||
| 83 | do { | ||||
| 84 | uint8_t Byte = Value & 0x7f; | ||||
| 85 | Value >>= 7; | ||||
| 86 | Count++; | ||||
| 87 | if (Value != 0 || Count < PadTo) | ||||
| 88 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
| 89 | OS << char(Byte); | ||||
| 90 | } while (Value != 0); | ||||
| 91 | |||||
| 92 | // Pad with 0x80 and emit a null byte at the end. | ||||
| 93 | if (Count < PadTo) { | ||||
| 94 | for (; Count < PadTo - 1; ++Count) | ||||
| 95 | OS << '\x80'; | ||||
| 96 | OS << '\x00'; | ||||
| 97 | Count++; | ||||
| 98 | } | ||||
| 99 | return Count; | ||||
| 100 | } | ||||
| 101 | |||||
| 102 | /// Utility function to encode a ULEB128 value to a buffer. Returns | ||||
| 103 | /// the length in bytes of the encoded value. | ||||
| 104 | inline unsigned encodeULEB128(uint64_t Value, uint8_t *p, | ||||
| 105 | unsigned PadTo = 0) { | ||||
| 106 | uint8_t *orig_p = p; | ||||
| 107 | unsigned Count = 0; | ||||
| 108 | do { | ||||
| 109 | uint8_t Byte = Value & 0x7f; | ||||
| 110 | Value >>= 7; | ||||
| 111 | Count++; | ||||
| 112 | if (Value != 0 || Count < PadTo) | ||||
| 113 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
| 114 | *p++ = Byte; | ||||
| 115 | } while (Value != 0); | ||||
| 116 | |||||
| 117 | // Pad with 0x80 and emit a null byte at the end. | ||||
| 118 | if (Count < PadTo) { | ||||
| 119 | for (; Count < PadTo - 1; ++Count) | ||||
| 120 | *p++ = '\x80'; | ||||
| 121 | *p++ = '\x00'; | ||||
| 122 | } | ||||
| 123 | |||||
| 124 | return (unsigned)(p - orig_p); | ||||
| 125 | } | ||||
| 126 | |||||
| 127 | /// Utility function to decode a ULEB128 value. | ||||
| 128 | inline uint64_t decodeULEB128(const uint8_t *p, unsigned *n = nullptr, | ||||
| 129 | const uint8_t *end = nullptr, | ||||
| 130 | const char **error = nullptr) { | ||||
| 131 | const uint8_t *orig_p = p; | ||||
| 132 | uint64_t Value = 0; | ||||
| 133 | unsigned Shift = 0; | ||||
| 134 | if (error
| ||||
| 135 | *error = nullptr; | ||||
| 136 | do { | ||||
| 137 | if (p == end) { | ||||
| 138 | if (error
| ||||
| 139 | *error = "malformed uleb128, extends past end"; | ||||
| 140 | if (n
| ||||
| 141 | *n = (unsigned)(p - orig_p); | ||||
| 142 | return 0; | ||||
| 143 | } | ||||
| 144 | uint64_t Slice = *p & 0x7f; | ||||
| 145 | if ((Shift >= 64 && Slice != 0) || Slice << Shift >> Shift != Slice) { | ||||
| 146 | if (error) | ||||
| 147 | *error = "uleb128 too big for uint64"; | ||||
| 148 | if (n) | ||||
| 149 | *n = (unsigned)(p - orig_p); | ||||
| 150 | return 0; | ||||
| 151 | } | ||||
| 152 | Value += Slice << Shift; | ||||
| 153 | Shift += 7; | ||||
| 154 | } while (*p++ >= 128); | ||||
| 155 | if (n) | ||||
| 156 | *n = (unsigned)(p - orig_p); | ||||
| 157 | return Value; | ||||
| 158 | } | ||||
| 159 | |||||
| 160 | /// Utility function to decode a SLEB128 value. | ||||
| 161 | inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr, | ||||
| 162 | const uint8_t *end = nullptr, | ||||
| 163 | const char **error = nullptr) { | ||||
| 164 | const uint8_t *orig_p = p; | ||||
| 165 | int64_t Value = 0; | ||||
| 166 | unsigned Shift = 0; | ||||
| 167 | uint8_t Byte; | ||||
| 168 | if (error) | ||||
| 169 | *error = nullptr; | ||||
| 170 | do { | ||||
| 171 | if (p == end) { | ||||
| 172 | if (error) | ||||
| 173 | *error = "malformed sleb128, extends past end"; | ||||
| 174 | if (n) | ||||
| 175 | *n = (unsigned)(p - orig_p); | ||||
| 176 | return 0; | ||||
| 177 | } | ||||
| 178 | Byte = *p; | ||||
| 179 | uint64_t Slice = Byte & 0x7f; | ||||
| 180 | if ((Shift >= 64 && Slice != (Value < 0 ? 0x7f : 0x00)) || | ||||
| 181 | (Shift == 63 && Slice != 0 && Slice != 0x7f)) { | ||||
| 182 | if (error) | ||||
| 183 | *error = "sleb128 too big for int64"; | ||||
| 184 | if (n) | ||||
| 185 | *n = (unsigned)(p - orig_p); | ||||
| 186 | return 0; | ||||
| 187 | } | ||||
| 188 | Value |= Slice << Shift; | ||||
| 189 | Shift += 7; | ||||
| 190 | ++p; | ||||
| 191 | } while (Byte >= 128); | ||||
| 192 | // Sign extend negative numbers if needed. | ||||
| 193 | if (Shift < 64 && (Byte & 0x40)) | ||||
| 194 | Value |= (-1ULL) << Shift; | ||||
| 195 | if (n) | ||||
| 196 | *n = (unsigned)(p - orig_p); | ||||
| 197 | return Value; | ||||
| 198 | } | ||||
| 199 | |||||
| 200 | /// Utility function to get the size of the ULEB128-encoded value. | ||||
| 201 | extern unsigned getULEB128Size(uint64_t Value); | ||||
| 202 | |||||
| 203 | /// Utility function to get the size of the SLEB128-encoded value. | ||||
| 204 | extern unsigned getSLEB128Size(int64_t Value); | ||||
| 205 | |||||
| 206 | } // namespace llvm | ||||
| 207 | |||||
| 208 | #endif // LLVM_SUPPORT_LEB128_H |