Bug Summary

File:llvm/lib/ProfileData/InstrProf.cpp
Warning:line 536, column 24
Dereference of null pointer (loaded from variable 'P')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name InstrProf.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/ProfileData -I /build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/llvm/lib/ProfileData -I include -I /build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-01-27-232948-117823-1 -x c++ /build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/llvm/lib/ProfileData/InstrProf.cpp

/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/llvm/lib/ProfileData/InstrProf.cpp

1//===- InstrProf.cpp - Instrumented profiling format support --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for clang's instrumentation based PGO and
10// coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProf.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallString.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Triple.h"
21#include "llvm/Config/config.h"
22#include "llvm/IR/Constant.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/Instruction.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/IR/Metadata.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/Type.h"
33#include "llvm/ProfileData/InstrProfReader.h"
34#include "llvm/Support/Casting.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/Compiler.h"
37#include "llvm/Support/Compression.h"
38#include "llvm/Support/Endian.h"
39#include "llvm/Support/Error.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/LEB128.h"
42#include "llvm/Support/ManagedStatic.h"
43#include "llvm/Support/MathExtras.h"
44#include "llvm/Support/Path.h"
45#include "llvm/Support/SwapByteOrder.h"
46#include <algorithm>
47#include <cassert>
48#include <cstddef>
49#include <cstdint>
50#include <cstring>
51#include <memory>
52#include <string>
53#include <system_error>
54#include <utility>
55#include <vector>
56
57using namespace llvm;
58
59static cl::opt<bool> StaticFuncFullModulePrefix(
60 "static-func-full-module-prefix", cl::init(true), cl::Hidden,
61 cl::desc("Use full module build paths in the profile counter names for "
62 "static functions."));
63
64// This option is tailored to users that have different top-level directory in
65// profile-gen and profile-use compilation. Users need to specific the number
66// of levels to strip. A value larger than the number of directories in the
67// source file will strip all the directory names and only leave the basename.
68//
69// Note current ThinLTO module importing for the indirect-calls assumes
70// the source directory name not being stripped. A non-zero option value here
71// can potentially prevent some inter-module indirect-call-promotions.
72static cl::opt<unsigned> StaticFuncStripDirNamePrefix(
73 "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
74 cl::desc("Strip specified level of directory name from source path in "
75 "the profile counter name for static functions."));
76
77static std::string getInstrProfErrString(instrprof_error Err,
78 const std::string &ErrMsg = "") {
79 std::string Msg;
80 raw_string_ostream OS(Msg);
81
82 switch (Err) {
83 case instrprof_error::success:
84 OS << "success";
85 break;
86 case instrprof_error::eof:
87 OS << "end of File";
88 break;
89 case instrprof_error::unrecognized_format:
90 OS << "unrecognized instrumentation profile encoding format";
91 break;
92 case instrprof_error::bad_magic:
93 OS << "invalid instrumentation profile data (bad magic)";
94 break;
95 case instrprof_error::bad_header:
96 OS << "invalid instrumentation profile data (file header is corrupt)";
97 break;
98 case instrprof_error::unsupported_version:
99 OS << "unsupported instrumentation profile format version";
100 break;
101 case instrprof_error::unsupported_hash_type:
102 OS << "unsupported instrumentation profile hash type";
103 break;
104 case instrprof_error::too_large:
105 OS << "too much profile data";
106 break;
107 case instrprof_error::truncated:
108 OS << "truncated profile data";
109 break;
110 case instrprof_error::malformed:
111 OS << "malformed instrumentation profile data";
112 break;
113 case instrprof_error::missing_debug_info_for_correlation:
114 OS << "debug info for correlation is required";
115 break;
116 case instrprof_error::unexpected_debug_info_for_correlation:
117 OS << "debug info for correlation is not necessary";
118 break;
119 case instrprof_error::unable_to_correlate_profile:
120 OS << "unable to correlate profile";
121 break;
122 case instrprof_error::invalid_prof:
123 OS << "invalid profile created. Please file a bug "
124 "at: " BUG_REPORT_URL"https://github.com/llvm/llvm-project/issues/"
125 " and include the profraw files that caused this error.";
126 break;
127 case instrprof_error::unknown_function:
128 OS << "no profile data available for function";
129 break;
130 case instrprof_error::hash_mismatch:
131 OS << "function control flow change detected (hash mismatch)";
132 break;
133 case instrprof_error::count_mismatch:
134 OS << "function basic block count change detected (counter mismatch)";
135 break;
136 case instrprof_error::counter_overflow:
137 OS << "counter overflow";
138 break;
139 case instrprof_error::value_site_count_mismatch:
140 OS << "function value site count change detected (counter mismatch)";
141 break;
142 case instrprof_error::compress_failed:
143 OS << "failed to compress data (zlib)";
144 break;
145 case instrprof_error::uncompress_failed:
146 OS << "failed to uncompress data (zlib)";
147 break;
148 case instrprof_error::empty_raw_profile:
149 OS << "empty raw profile file";
150 break;
151 case instrprof_error::zlib_unavailable:
152 OS << "profile uses zlib compression but the profile reader was built "
153 "without zlib support";
154 break;
155 }
156
157 // If optional error message is not empty, append it to the message.
158 if (!ErrMsg.empty())
159 OS << ": " << ErrMsg;
160
161 return OS.str();
162}
163
164namespace {
165
166// FIXME: This class is only here to support the transition to llvm::Error. It
167// will be removed once this transition is complete. Clients should prefer to
168// deal with the Error value directly, rather than converting to error_code.
169class InstrProfErrorCategoryType : public std::error_category {
170 const char *name() const noexcept override { return "llvm.instrprof"; }
171
172 std::string message(int IE) const override {
173 return getInstrProfErrString(static_cast<instrprof_error>(IE));
174 }
175};
176
177} // end anonymous namespace
178
179static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
180
181const std::error_category &llvm::instrprof_category() {
182 return *ErrorCategory;
183}
184
185namespace {
186
187const char *InstrProfSectNameCommon[] = {
188#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
189 SectNameCommon,
190#include "llvm/ProfileData/InstrProfData.inc"
191};
192
193const char *InstrProfSectNameCoff[] = {
194#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
195 SectNameCoff,
196#include "llvm/ProfileData/InstrProfData.inc"
197};
198
199const char *InstrProfSectNamePrefix[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
201 Prefix,
202#include "llvm/ProfileData/InstrProfData.inc"
203};
204
205} // namespace
206
207namespace llvm {
208
209cl::opt<bool> DoInstrProfNameCompression(
210 "enable-name-compression",
211 cl::desc("Enable name/filename string compression"), cl::init(true));
212
213std::string getInstrProfSectionName(InstrProfSectKind IPSK,
214 Triple::ObjectFormatType OF,
215 bool AddSegmentInfo) {
216 std::string SectName;
217
218 if (OF == Triple::MachO && AddSegmentInfo)
219 SectName = InstrProfSectNamePrefix[IPSK];
220
221 if (OF == Triple::COFF)
222 SectName += InstrProfSectNameCoff[IPSK];
223 else
224 SectName += InstrProfSectNameCommon[IPSK];
225
226 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
227 SectName += ",regular,live_support";
228
229 return SectName;
230}
231
232void SoftInstrProfErrors::addError(instrprof_error IE) {
233 if (IE == instrprof_error::success)
234 return;
235
236 if (FirstError == instrprof_error::success)
237 FirstError = IE;
238
239 switch (IE) {
240 case instrprof_error::hash_mismatch:
241 ++NumHashMismatches;
242 break;
243 case instrprof_error::count_mismatch:
244 ++NumCountMismatches;
245 break;
246 case instrprof_error::counter_overflow:
247 ++NumCounterOverflows;
248 break;
249 case instrprof_error::value_site_count_mismatch:
250 ++NumValueSiteCountMismatches;
251 break;
252 default:
253 llvm_unreachable("Not a soft error")::llvm::llvm_unreachable_internal("Not a soft error", "llvm/lib/ProfileData/InstrProf.cpp"
, 253)
;
254 }
255}
256
257std::string InstrProfError::message() const {
258 return getInstrProfErrString(Err, Msg);
259}
260
261char InstrProfError::ID = 0;
262
263std::string getPGOFuncName(StringRef RawFuncName,
264 GlobalValue::LinkageTypes Linkage,
265 StringRef FileName,
266 uint64_t Version LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__))) {
267 return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
268}
269
270// Strip NumPrefix level of directory name from PathNameStr. If the number of
271// directory separators is less than NumPrefix, strip all the directories and
272// leave base file name only.
273static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
274 uint32_t Count = NumPrefix;
275 uint32_t Pos = 0, LastPos = 0;
276 for (auto & CI : PathNameStr) {
277 ++Pos;
278 if (llvm::sys::path::is_separator(CI)) {
279 LastPos = Pos;
280 --Count;
281 }
282 if (Count == 0)
283 break;
284 }
285 return PathNameStr.substr(LastPos);
286}
287
288// Return the PGOFuncName. This function has some special handling when called
289// in LTO optimization. The following only applies when calling in LTO passes
290// (when \c InLTO is true): LTO's internalization privatizes many global linkage
291// symbols. This happens after value profile annotation, but those internal
292// linkage functions should not have a source prefix.
293// Additionally, for ThinLTO mode, exported internal functions are promoted
294// and renamed. We need to ensure that the original internal PGO name is
295// used when computing the GUID that is compared against the profiled GUIDs.
296// To differentiate compiler generated internal symbols from original ones,
297// PGOFuncName meta data are created and attached to the original internal
298// symbols in the value profile annotation step
299// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
300// data, its original linkage must be non-internal.
301std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
302 if (!InLTO) {
303 StringRef FileName(F.getParent()->getSourceFileName());
304 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
305 if (StripLevel < StaticFuncStripDirNamePrefix)
306 StripLevel = StaticFuncStripDirNamePrefix;
307 if (StripLevel)
308 FileName = stripDirPrefix(FileName, StripLevel);
309 return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
310 }
311
312 // In LTO mode (when InLTO is true), first check if there is a meta data.
313 if (MDNode *MD = getPGOFuncNameMetadata(F)) {
314 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
315 return S.str();
316 }
317
318 // If there is no meta data, the function must be a global before the value
319 // profile annotation pass. Its current linkage may be internal if it is
320 // internalized in LTO mode.
321 return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
322}
323
324StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
325 if (FileName.empty())
326 return PGOFuncName;
327 // Drop the file name including ':'. See also getPGOFuncName.
328 if (PGOFuncName.startswith(FileName))
329 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
330 return PGOFuncName;
331}
332
333// \p FuncName is the string used as profile lookup key for the function. A
334// symbol is created to hold the name. Return the legalized symbol name.
335std::string getPGOFuncNameVarName(StringRef FuncName,
336 GlobalValue::LinkageTypes Linkage) {
337 std::string VarName = std::string(getInstrProfNameVarPrefix());
338 VarName += FuncName;
339
340 if (!GlobalValue::isLocalLinkage(Linkage))
341 return VarName;
342
343 // Now fix up illegal chars in local VarName that may upset the assembler.
344 const char *InvalidChars = "-:<>/\"'";
345 size_t found = VarName.find_first_of(InvalidChars);
346 while (found != std::string::npos) {
347 VarName[found] = '_';
348 found = VarName.find_first_of(InvalidChars, found + 1);
349 }
350 return VarName;
351}
352
353GlobalVariable *createPGOFuncNameVar(Module &M,
354 GlobalValue::LinkageTypes Linkage,
355 StringRef PGOFuncName) {
356 // We generally want to match the function's linkage, but available_externally
357 // and extern_weak both have the wrong semantics, and anything that doesn't
358 // need to link across compilation units doesn't need to be visible at all.
359 if (Linkage == GlobalValue::ExternalWeakLinkage)
360 Linkage = GlobalValue::LinkOnceAnyLinkage;
361 else if (Linkage == GlobalValue::AvailableExternallyLinkage)
362 Linkage = GlobalValue::LinkOnceODRLinkage;
363 else if (Linkage == GlobalValue::InternalLinkage ||
364 Linkage == GlobalValue::ExternalLinkage)
365 Linkage = GlobalValue::PrivateLinkage;
366
367 auto *Value =
368 ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
369 auto FuncNameVar =
370 new GlobalVariable(M, Value->getType(), true, Linkage, Value,
371 getPGOFuncNameVarName(PGOFuncName, Linkage));
372
373 // Hide the symbol so that we correctly get a copy for each executable.
374 if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
375 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
376
377 return FuncNameVar;
378}
379
380GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
381 return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
382}
383
384Error InstrProfSymtab::create(Module &M, bool InLTO) {
385 for (Function &F : M) {
386 // Function may not have a name: like using asm("") to overwrite the name.
387 // Ignore in this case.
388 if (!F.hasName())
389 continue;
390 const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
391 if (Error E = addFuncName(PGOFuncName))
392 return E;
393 MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
394 // In ThinLTO, local function may have been promoted to global and have
395 // suffix ".llvm." added to the function name. We need to add the
396 // stripped function name to the symbol table so that we can find a match
397 // from profile.
398 //
399 // We may have other suffixes similar as ".llvm." which are needed to
400 // be stripped before the matching, but ".__uniq." suffix which is used
401 // to differentiate internal linkage functions in different modules
402 // should be kept. Now this is the only suffix with the pattern ".xxx"
403 // which is kept before matching.
404 const std::string UniqSuffix = ".__uniq.";
405 auto pos = PGOFuncName.find(UniqSuffix);
406 // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
407 // search '.' from the beginning.
408 if (pos != std::string::npos)
409 pos += UniqSuffix.length();
410 else
411 pos = 0;
412 pos = PGOFuncName.find('.', pos);
413 if (pos != std::string::npos && pos != 0) {
414 const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
415 if (Error E = addFuncName(OtherFuncName))
416 return E;
417 MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
418 }
419 }
420 Sorted = false;
421 finalizeSymtab();
422 return Error::success();
423}
424
425uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
426 finalizeSymtab();
427 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
428 return A.first < Address;
429 });
430 // Raw function pointer collected by value profiler may be from
431 // external functions that are not instrumented. They won't have
432 // mapping data to be used by the deserializer. Force the value to
433 // be 0 in this case.
434 if (It != AddrToMD5Map.end() && It->first == Address)
435 return (uint64_t)It->second;
436 return 0;
437}
438
439Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
440 bool doCompression, std::string &Result) {
441 assert(!NameStrs.empty() && "No name data to emit")(static_cast <bool> (!NameStrs.empty() && "No name data to emit"
) ? void (0) : __assert_fail ("!NameStrs.empty() && \"No name data to emit\""
, "llvm/lib/ProfileData/InstrProf.cpp", 441, __extension__ __PRETTY_FUNCTION__
))
;
442
443 uint8_t Header[16], *P = Header;
444 std::string UncompressedNameStrings =
445 join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
446
447 assert(StringRef(UncompressedNameStrings)(static_cast <bool> (StringRef(UncompressedNameStrings)
.count(getInstrProfNameSeparator()) == (NameStrs.size() - 1)
&& "PGO name is invalid (contains separator token)")
? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "llvm/lib/ProfileData/InstrProf.cpp", 449, __extension__ __PRETTY_FUNCTION__
))
448 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&(static_cast <bool> (StringRef(UncompressedNameStrings)
.count(getInstrProfNameSeparator()) == (NameStrs.size() - 1)
&& "PGO name is invalid (contains separator token)")
? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "llvm/lib/ProfileData/InstrProf.cpp", 449, __extension__ __PRETTY_FUNCTION__
))
449 "PGO name is invalid (contains separator token)")(static_cast <bool> (StringRef(UncompressedNameStrings)
.count(getInstrProfNameSeparator()) == (NameStrs.size() - 1)
&& "PGO name is invalid (contains separator token)")
? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "llvm/lib/ProfileData/InstrProf.cpp", 449, __extension__ __PRETTY_FUNCTION__
))
;
450
451 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
452 P += EncLen;
453
454 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
455 EncLen = encodeULEB128(CompressedLen, P);
456 P += EncLen;
457 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
458 unsigned HeaderLen = P - &Header[0];
459 Result.append(HeaderStr, HeaderLen);
460 Result += InputStr;
461 return Error::success();
462 };
463
464 if (!doCompression) {
465 return WriteStringToResult(0, UncompressedNameStrings);
466 }
467
468 SmallString<128> CompressedNameStrings;
469 Error E = zlib::compress(StringRef(UncompressedNameStrings),
470 CompressedNameStrings, zlib::BestSizeCompression);
471 if (E) {
472 consumeError(std::move(E));
473 return make_error<InstrProfError>(instrprof_error::compress_failed);
474 }
475
476 return WriteStringToResult(CompressedNameStrings.size(),
477 CompressedNameStrings);
478}
479
480StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
481 auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
482 StringRef NameStr =
483 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
484 return NameStr;
485}
486
487Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
488 std::string &Result, bool doCompression) {
489 std::vector<std::string> NameStrs;
490 for (auto *NameVar : NameVars) {
491 NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
492 }
493 return collectPGOFuncNameStrings(
494 NameStrs, zlib::isAvailable() && doCompression, Result);
495}
496
497Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
498 const uint8_t *P = NameStrings.bytes_begin();
499 const uint8_t *EndP = NameStrings.bytes_end();
500 while (P < EndP) {
1
Assuming 'P' is < 'EndP'
2
Loop condition is true. Entering loop body
501 uint32_t N;
502 uint64_t UncompressedSize = decodeULEB128(P, &N);
3
Calling 'decodeULEB128'
10
Returning from 'decodeULEB128'
11
'UncompressedSize' initialized to 0
503 P += N;
504 uint64_t CompressedSize = decodeULEB128(P, &N);
505 P += N;
506 bool isCompressed = (CompressedSize != 0);
507 SmallString<128> UncompressedNameStrings;
508 StringRef NameStrings;
509 if (isCompressed
11.1
'isCompressed' is false
11.1
'isCompressed' is false
) {
12
Taking false branch
510 if (!llvm::zlib::isAvailable())
511 return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
512
513 StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
514 CompressedSize);
515 if (Error E =
516 zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
517 UncompressedSize)) {
518 consumeError(std::move(E));
519 return make_error<InstrProfError>(instrprof_error::uncompress_failed);
520 }
521 P += CompressedSize;
522 NameStrings = StringRef(UncompressedNameStrings.data(),
523 UncompressedNameStrings.size());
524 } else {
525 NameStrings =
526 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
527 P += UncompressedSize;
13
Null pointer value stored to 'P'
528 }
529 // Now parse the name strings.
530 SmallVector<StringRef, 0> Names;
531 NameStrings.split(Names, getInstrProfNameSeparator());
532 for (StringRef &Name : Names)
14
Assuming '__begin2' is equal to '__end2'
533 if (Error E = Symtab.addFuncName(Name))
534 return E;
535
536 while (P
14.1
'P' is < 'EndP'
14.1
'P' is < 'EndP'
< EndP && *P == 0)
15
Dereference of null pointer (loaded from variable 'P')
537 P++;
538 }
539 return Error::success();
540}
541
542void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
543 uint64_t FuncSum = 0;
544 Sum.NumEntries += Counts.size();
545 for (uint64_t Count : Counts)
546 FuncSum += Count;
547 Sum.CountSum += FuncSum;
548
549 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
550 uint64_t KindSum = 0;
551 uint32_t NumValueSites = getNumValueSites(VK);
552 for (size_t I = 0; I < NumValueSites; ++I) {
553 uint32_t NV = getNumValueDataForSite(VK, I);
554 std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I);
555 for (uint32_t V = 0; V < NV; V++)
556 KindSum += VD[V].Count;
557 }
558 Sum.ValueCounts[VK] += KindSum;
559 }
560}
561
562void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input,
563 uint32_t ValueKind,
564 OverlapStats &Overlap,
565 OverlapStats &FuncLevelOverlap) {
566 this->sortByTargetValues();
567 Input.sortByTargetValues();
568 double Score = 0.0f, FuncLevelScore = 0.0f;
569 auto I = ValueData.begin();
570 auto IE = ValueData.end();
571 auto J = Input.ValueData.begin();
572 auto JE = Input.ValueData.end();
573 while (I != IE && J != JE) {
574 if (I->Value == J->Value) {
575 Score += OverlapStats::score(I->Count, J->Count,
576 Overlap.Base.ValueCounts[ValueKind],
577 Overlap.Test.ValueCounts[ValueKind]);
578 FuncLevelScore += OverlapStats::score(
579 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
580 FuncLevelOverlap.Test.ValueCounts[ValueKind]);
581 ++I;
582 } else if (I->Value < J->Value) {
583 ++I;
584 continue;
585 }
586 ++J;
587 }
588 Overlap.Overlap.ValueCounts[ValueKind] += Score;
589 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
590}
591
592// Return false on mismatch.
593void InstrProfRecord::overlapValueProfData(uint32_t ValueKind,
594 InstrProfRecord &Other,
595 OverlapStats &Overlap,
596 OverlapStats &FuncLevelOverlap) {
597 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
598 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind))(static_cast <bool> (ThisNumValueSites == Other.getNumValueSites
(ValueKind)) ? void (0) : __assert_fail ("ThisNumValueSites == Other.getNumValueSites(ValueKind)"
, "llvm/lib/ProfileData/InstrProf.cpp", 598, __extension__ __PRETTY_FUNCTION__
))
;
599 if (!ThisNumValueSites)
600 return;
601
602 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
603 getOrCreateValueSitesForKind(ValueKind);
604 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
605 Other.getValueSitesForKind(ValueKind);
606 for (uint32_t I = 0; I < ThisNumValueSites; I++)
607 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
608 FuncLevelOverlap);
609}
610
611void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap,
612 OverlapStats &FuncLevelOverlap,
613 uint64_t ValueCutoff) {
614 // FuncLevel CountSum for other should already computed and nonzero.
615 assert(FuncLevelOverlap.Test.CountSum >= 1.0f)(static_cast <bool> (FuncLevelOverlap.Test.CountSum >=
1.0f) ? void (0) : __assert_fail ("FuncLevelOverlap.Test.CountSum >= 1.0f"
, "llvm/lib/ProfileData/InstrProf.cpp", 615, __extension__ __PRETTY_FUNCTION__
))
;
616 accumulateCounts(FuncLevelOverlap.Base);
617 bool Mismatch = (Counts.size() != Other.Counts.size());
618
619 // Check if the value profiles mismatch.
620 if (!Mismatch) {
621 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
622 uint32_t ThisNumValueSites = getNumValueSites(Kind);
623 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
624 if (ThisNumValueSites != OtherNumValueSites) {
625 Mismatch = true;
626 break;
627 }
628 }
629 }
630 if (Mismatch) {
631 Overlap.addOneMismatch(FuncLevelOverlap.Test);
632 return;
633 }
634
635 // Compute overlap for value counts.
636 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
637 overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
638
639 double Score = 0.0;
640 uint64_t MaxCount = 0;
641 // Compute overlap for edge counts.
642 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
643 Score += OverlapStats::score(Counts[I], Other.Counts[I],
644 Overlap.Base.CountSum, Overlap.Test.CountSum);
645 MaxCount = std::max(Other.Counts[I], MaxCount);
646 }
647 Overlap.Overlap.CountSum += Score;
648 Overlap.Overlap.NumEntries += 1;
649
650 if (MaxCount >= ValueCutoff) {
651 double FuncScore = 0.0;
652 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
653 FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
654 FuncLevelOverlap.Base.CountSum,
655 FuncLevelOverlap.Test.CountSum);
656 FuncLevelOverlap.Overlap.CountSum = FuncScore;
657 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
658 FuncLevelOverlap.Valid = true;
659 }
660}
661
662void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
663 uint64_t Weight,
664 function_ref<void(instrprof_error)> Warn) {
665 this->sortByTargetValues();
666 Input.sortByTargetValues();
667 auto I = ValueData.begin();
668 auto IE = ValueData.end();
669 for (const InstrProfValueData &J : Input.ValueData) {
670 while (I != IE && I->Value < J.Value)
671 ++I;
672 if (I != IE && I->Value == J.Value) {
673 bool Overflowed;
674 I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed);
675 if (Overflowed)
676 Warn(instrprof_error::counter_overflow);
677 ++I;
678 continue;
679 }
680 ValueData.insert(I, J);
681 }
682}
683
684void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D,
685 function_ref<void(instrprof_error)> Warn) {
686 for (InstrProfValueData &I : ValueData) {
687 bool Overflowed;
688 I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
689 if (Overflowed)
690 Warn(instrprof_error::counter_overflow);
691 }
692}
693
694// Merge Value Profile data from Src record to this record for ValueKind.
695// Scale merged value counts by \p Weight.
696void InstrProfRecord::mergeValueProfData(
697 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
698 function_ref<void(instrprof_error)> Warn) {
699 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
700 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
701 if (ThisNumValueSites != OtherNumValueSites) {
702 Warn(instrprof_error::value_site_count_mismatch);
703 return;
704 }
705 if (!ThisNumValueSites)
706 return;
707 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
708 getOrCreateValueSitesForKind(ValueKind);
709 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
710 Src.getValueSitesForKind(ValueKind);
711 for (uint32_t I = 0; I < ThisNumValueSites; I++)
712 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
713}
714
715void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
716 function_ref<void(instrprof_error)> Warn) {
717 // If the number of counters doesn't match we either have bad data
718 // or a hash collision.
719 if (Counts.size() != Other.Counts.size()) {
720 Warn(instrprof_error::count_mismatch);
721 return;
722 }
723
724 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
725 bool Overflowed;
726 Counts[I] =
727 SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
728 if (Overflowed)
729 Warn(instrprof_error::counter_overflow);
730 }
731
732 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
733 mergeValueProfData(Kind, Other, Weight, Warn);
734}
735
736void InstrProfRecord::scaleValueProfData(
737 uint32_t ValueKind, uint64_t N, uint64_t D,
738 function_ref<void(instrprof_error)> Warn) {
739 for (auto &R : getValueSitesForKind(ValueKind))
740 R.scale(N, D, Warn);
741}
742
743void InstrProfRecord::scale(uint64_t N, uint64_t D,
744 function_ref<void(instrprof_error)> Warn) {
745 assert(D != 0 && "D cannot be 0")(static_cast <bool> (D != 0 && "D cannot be 0")
? void (0) : __assert_fail ("D != 0 && \"D cannot be 0\""
, "llvm/lib/ProfileData/InstrProf.cpp", 745, __extension__ __PRETTY_FUNCTION__
))
;
746 for (auto &Count : this->Counts) {
747 bool Overflowed;
748 Count = SaturatingMultiply(Count, N, &Overflowed) / D;
749 if (Overflowed)
750 Warn(instrprof_error::counter_overflow);
751 }
752 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
753 scaleValueProfData(Kind, N, D, Warn);
754}
755
756// Map indirect call target name hash to name string.
757uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
758 InstrProfSymtab *SymTab) {
759 if (!SymTab)
760 return Value;
761
762 if (ValueKind == IPVK_IndirectCallTarget)
763 return SymTab->getFunctionHashFromAddress(Value);
764
765 return Value;
766}
767
768void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
769 InstrProfValueData *VData, uint32_t N,
770 InstrProfSymtab *ValueMap) {
771 for (uint32_t I = 0; I < N; I++) {
772 VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
773 }
774 std::vector<InstrProfValueSiteRecord> &ValueSites =
775 getOrCreateValueSitesForKind(ValueKind);
776 if (N == 0)
777 ValueSites.emplace_back();
778 else
779 ValueSites.emplace_back(VData, VData + N);
780}
781
782#define INSTR_PROF_COMMON_API_IMPL
783#include "llvm/ProfileData/InstrProfData.inc"
784
785/*!
786 * ValueProfRecordClosure Interface implementation for InstrProfRecord
787 * class. These C wrappers are used as adaptors so that C++ code can be
788 * invoked as callbacks.
789 */
790uint32_t getNumValueKindsInstrProf(const void *Record) {
791 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
792}
793
794uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
795 return reinterpret_cast<const InstrProfRecord *>(Record)
796 ->getNumValueSites(VKind);
797}
798
799uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
800 return reinterpret_cast<const InstrProfRecord *>(Record)
801 ->getNumValueData(VKind);
802}
803
804uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
805 uint32_t S) {
806 return reinterpret_cast<const InstrProfRecord *>(R)
807 ->getNumValueDataForSite(VK, S);
808}
809
810void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
811 uint32_t K, uint32_t S) {
812 reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
813}
814
815ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
816 ValueProfData *VD =
817 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
818 memset(VD, 0, TotalSizeInBytes);
819 return VD;
820}
821
822static ValueProfRecordClosure InstrProfRecordClosure = {
823 nullptr,
824 getNumValueKindsInstrProf,
825 getNumValueSitesInstrProf,
826 getNumValueDataInstrProf,
827 getNumValueDataForSiteInstrProf,
828 nullptr,
829 getValueForSiteInstrProf,
830 allocValueProfDataInstrProf};
831
832// Wrapper implementation using the closure mechanism.
833uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
834 auto Closure = InstrProfRecordClosure;
835 Closure.Record = &Record;
836 return getValueProfDataSize(&Closure);
837}
838
839// Wrapper implementation using the closure mechanism.
840std::unique_ptr<ValueProfData>
841ValueProfData::serializeFrom(const InstrProfRecord &Record) {
842 InstrProfRecordClosure.Record = &Record;
843
844 std::unique_ptr<ValueProfData> VPD(
845 serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
846 return VPD;
847}
848
849void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
850 InstrProfSymtab *SymTab) {
851 Record.reserveSites(Kind, NumValueSites);
852
853 InstrProfValueData *ValueData = getValueProfRecordValueData(this);
854 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
855 uint8_t ValueDataCount = this->SiteCountArray[VSite];
856 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
857 ValueData += ValueDataCount;
858 }
859}
860
861// For writing/serializing, Old is the host endianness, and New is
862// byte order intended on disk. For Reading/deserialization, Old
863// is the on-disk source endianness, and New is the host endianness.
864void ValueProfRecord::swapBytes(support::endianness Old,
865 support::endianness New) {
866 using namespace support;
867
868 if (Old == New)
869 return;
870
871 if (getHostEndianness() != Old) {
872 sys::swapByteOrder<uint32_t>(NumValueSites);
873 sys::swapByteOrder<uint32_t>(Kind);
874 }
875 uint32_t ND = getValueProfRecordNumValueData(this);
876 InstrProfValueData *VD = getValueProfRecordValueData(this);
877
878 // No need to swap byte array: SiteCountArrray.
879 for (uint32_t I = 0; I < ND; I++) {
880 sys::swapByteOrder<uint64_t>(VD[I].Value);
881 sys::swapByteOrder<uint64_t>(VD[I].Count);
882 }
883 if (getHostEndianness() == Old) {
884 sys::swapByteOrder<uint32_t>(NumValueSites);
885 sys::swapByteOrder<uint32_t>(Kind);
886 }
887}
888
889void ValueProfData::deserializeTo(InstrProfRecord &Record,
890 InstrProfSymtab *SymTab) {
891 if (NumValueKinds == 0)
892 return;
893
894 ValueProfRecord *VR = getFirstValueProfRecord(this);
895 for (uint32_t K = 0; K < NumValueKinds; K++) {
896 VR->deserializeTo(Record, SymTab);
897 VR = getValueProfRecordNext(VR);
898 }
899}
900
901template <class T>
902static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
903 using namespace support;
904
905 if (Orig == little)
906 return endian::readNext<T, little, unaligned>(D);
907 else
908 return endian::readNext<T, big, unaligned>(D);
909}
910
911static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
912 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
913 ValueProfData());
914}
915
916Error ValueProfData::checkIntegrity() {
917 if (NumValueKinds > IPVK_Last + 1)
918 return make_error<InstrProfError>(
919 instrprof_error::malformed, "number of value profile kinds is invalid");
920 // Total size needs to be multiple of quadword size.
921 if (TotalSize % sizeof(uint64_t))
922 return make_error<InstrProfError>(
923 instrprof_error::malformed, "total size is not multiples of quardword");
924
925 ValueProfRecord *VR = getFirstValueProfRecord(this);
926 for (uint32_t K = 0; K < this->NumValueKinds; K++) {
927 if (VR->Kind > IPVK_Last)
928 return make_error<InstrProfError>(instrprof_error::malformed,
929 "value kind is invalid");
930 VR = getValueProfRecordNext(VR);
931 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
932 return make_error<InstrProfError>(
933 instrprof_error::malformed,
934 "value profile address is greater than total size");
935 }
936 return Error::success();
937}
938
939Expected<std::unique_ptr<ValueProfData>>
940ValueProfData::getValueProfData(const unsigned char *D,
941 const unsigned char *const BufferEnd,
942 support::endianness Endianness) {
943 using namespace support;
944
945 if (D + sizeof(ValueProfData) > BufferEnd)
946 return make_error<InstrProfError>(instrprof_error::truncated);
947
948 const unsigned char *Header = D;
949 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
950 if (D + TotalSize > BufferEnd)
951 return make_error<InstrProfError>(instrprof_error::too_large);
952
953 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
954 memcpy(VPD.get(), D, TotalSize);
955 // Byte swap.
956 VPD->swapBytesToHost(Endianness);
957
958 Error E = VPD->checkIntegrity();
959 if (E)
960 return std::move(E);
961
962 return std::move(VPD);
963}
964
965void ValueProfData::swapBytesToHost(support::endianness Endianness) {
966 using namespace support;
967
968 if (Endianness == getHostEndianness())
969 return;
970
971 sys::swapByteOrder<uint32_t>(TotalSize);
972 sys::swapByteOrder<uint32_t>(NumValueKinds);
973
974 ValueProfRecord *VR = getFirstValueProfRecord(this);
975 for (uint32_t K = 0; K < NumValueKinds; K++) {
976 VR->swapBytes(Endianness, getHostEndianness());
977 VR = getValueProfRecordNext(VR);
978 }
979}
980
981void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
982 using namespace support;
983
984 if (Endianness == getHostEndianness())
985 return;
986
987 ValueProfRecord *VR = getFirstValueProfRecord(this);
988 for (uint32_t K = 0; K < NumValueKinds; K++) {
989 ValueProfRecord *NVR = getValueProfRecordNext(VR);
990 VR->swapBytes(getHostEndianness(), Endianness);
991 VR = NVR;
992 }
993 sys::swapByteOrder<uint32_t>(TotalSize);
994 sys::swapByteOrder<uint32_t>(NumValueKinds);
995}
996
997void annotateValueSite(Module &M, Instruction &Inst,
998 const InstrProfRecord &InstrProfR,
999 InstrProfValueKind ValueKind, uint32_t SiteIdx,
1000 uint32_t MaxMDCount) {
1001 uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
1002 if (!NV)
1003 return;
1004
1005 uint64_t Sum = 0;
1006 std::unique_ptr<InstrProfValueData[]> VD =
1007 InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
1008
1009 ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
1010 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1011}
1012
1013void annotateValueSite(Module &M, Instruction &Inst,
1014 ArrayRef<InstrProfValueData> VDs,
1015 uint64_t Sum, InstrProfValueKind ValueKind,
1016 uint32_t MaxMDCount) {
1017 LLVMContext &Ctx = M.getContext();
1018 MDBuilder MDHelper(Ctx);
1019 SmallVector<Metadata *, 3> Vals;
1020 // Tag
1021 Vals.push_back(MDHelper.createString("VP"));
1022 // Value Kind
1023 Vals.push_back(MDHelper.createConstant(
1024 ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
1025 // Total Count
1026 Vals.push_back(
1027 MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
1028
1029 // Value Profile Data
1030 uint32_t MDCount = MaxMDCount;
1031 for (auto &VD : VDs) {
1032 Vals.push_back(MDHelper.createConstant(
1033 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
1034 Vals.push_back(MDHelper.createConstant(
1035 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
1036 if (--MDCount == 0)
1037 break;
1038 }
1039 Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
1040}
1041
1042bool getValueProfDataFromInst(const Instruction &Inst,
1043 InstrProfValueKind ValueKind,
1044 uint32_t MaxNumValueData,
1045 InstrProfValueData ValueData[],
1046 uint32_t &ActualNumValueData, uint64_t &TotalC,
1047 bool GetNoICPValue) {
1048 MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1049 if (!MD)
1050 return false;
1051
1052 unsigned NOps = MD->getNumOperands();
1053
1054 if (NOps < 5)
1055 return false;
1056
1057 // Operand 0 is a string tag "VP":
1058 MDString *Tag = cast<MDString>(MD->getOperand(0));
1059 if (!Tag)
1060 return false;
1061
1062 if (!Tag->getString().equals("VP"))
1063 return false;
1064
1065 // Now check kind:
1066 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1067 if (!KindInt)
1068 return false;
1069 if (KindInt->getZExtValue() != ValueKind)
1070 return false;
1071
1072 // Get total count
1073 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1074 if (!TotalCInt)
1075 return false;
1076 TotalC = TotalCInt->getZExtValue();
1077
1078 ActualNumValueData = 0;
1079
1080 for (unsigned I = 3; I < NOps; I += 2) {
1081 if (ActualNumValueData >= MaxNumValueData)
1082 break;
1083 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1084 ConstantInt *Count =
1085 mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1086 if (!Value || !Count)
1087 return false;
1088 uint64_t CntValue = Count->getZExtValue();
1089 if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1090 continue;
1091 ValueData[ActualNumValueData].Value = Value->getZExtValue();
1092 ValueData[ActualNumValueData].Count = CntValue;
1093 ActualNumValueData++;
1094 }
1095 return true;
1096}
1097
1098MDNode *getPGOFuncNameMetadata(const Function &F) {
1099 return F.getMetadata(getPGOFuncNameMetadataName());
1100}
1101
1102void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
1103 // Only for internal linkage functions.
1104 if (PGOFuncName == F.getName())
1105 return;
1106 // Don't create duplicated meta-data.
1107 if (getPGOFuncNameMetadata(F))
1108 return;
1109 LLVMContext &C = F.getContext();
1110 MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1111 F.setMetadata(getPGOFuncNameMetadataName(), N);
1112}
1113
1114bool needsComdatForCounter(const Function &F, const Module &M) {
1115 if (F.hasComdat())
1116 return true;
1117
1118 if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1119 return false;
1120
1121 // See createPGOFuncNameVar for more details. To avoid link errors, profile
1122 // counters for function with available_externally linkage needs to be changed
1123 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1124 // created. Without using comdat, duplicate entries won't be removed by the
1125 // linker leading to increased data segement size and raw profile size. Even
1126 // worse, since the referenced counter from profile per-function data object
1127 // will be resolved to the common strong definition, the profile counts for
1128 // available_externally functions will end up being duplicated in raw profile
1129 // data. This can result in distorted profile as the counts of those dups
1130 // will be accumulated by the profile merger.
1131 GlobalValue::LinkageTypes Linkage = F.getLinkage();
1132 if (Linkage != GlobalValue::ExternalWeakLinkage &&
1133 Linkage != GlobalValue::AvailableExternallyLinkage)
1134 return false;
1135
1136 return true;
1137}
1138
1139// Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1140bool isIRPGOFlagSet(const Module *M) {
1141 auto IRInstrVar =
1142 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version");
1143 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1144 return false;
1145
1146 // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1147 // have the decl.
1148 if (IRInstrVar->isDeclaration())
1149 return true;
1150
1151 // Check if the flag is set.
1152 if (!IRInstrVar->hasInitializer())
1153 return false;
1154
1155 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1156 if (!InitVal)
1157 return false;
1158 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF(0x1ULL << 56)) != 0;
1159}
1160
1161// Check if we can safely rename this Comdat function.
1162bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1163 if (F.getName().empty())
1164 return false;
1165 if (!needsComdatForCounter(F, *(F.getParent())))
1166 return false;
1167 // Unsafe to rename the address-taken function (which can be used in
1168 // function comparison).
1169 if (CheckAddressTaken && F.hasAddressTaken())
1170 return false;
1171 // Only safe to do if this function may be discarded if it is not used
1172 // in the compilation unit.
1173 if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1174 return false;
1175
1176 // For AvailableExternallyLinkage functions.
1177 if (!F.hasComdat()) {
1178 assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage)(static_cast <bool> (F.getLinkage() == GlobalValue::AvailableExternallyLinkage
) ? void (0) : __assert_fail ("F.getLinkage() == GlobalValue::AvailableExternallyLinkage"
, "llvm/lib/ProfileData/InstrProf.cpp", 1178, __extension__ __PRETTY_FUNCTION__
))
;
1179 return true;
1180 }
1181 return true;
1182}
1183
1184// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1185// aware this is an ir_level profile so it can set the version flag.
1186GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS,
1187 bool InstrEntryBBEnabled,
1188 bool DebugInfoCorrelate) {
1189 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version");
1190 Type *IntTy64 = Type::getInt64Ty(M.getContext());
1191 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION8 | VARIANT_MASK_IR_PROF(0x1ULL << 56));
1192 if (IsCS)
1193 ProfileVersion |= VARIANT_MASK_CSIR_PROF(0x1ULL << 57);
1194 if (InstrEntryBBEnabled)
1195 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY(0x1ULL << 58);
1196 if (DebugInfoCorrelate)
1197 ProfileVersion |= VARIANT_MASK_DBG_CORRELATE(0x1ULL << 59);
1198 auto IRLevelVersionVariable = new GlobalVariable(
1199 M, IntTy64, true, GlobalValue::WeakAnyLinkage,
1200 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
1201 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
1202 Triple TT(M.getTargetTriple());
1203 if (TT.supportsCOMDAT()) {
1204 IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage);
1205 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
1206 }
1207 return IRLevelVersionVariable;
1208}
1209
1210// Create the variable for the profile file name.
1211void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1212 if (InstrProfileOutput.empty())
1213 return;
1214 Constant *ProfileNameConst =
1215 ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1216 GlobalVariable *ProfileNameVar = new GlobalVariable(
1217 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1218 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename");
1219 Triple TT(M.getTargetTriple());
1220 if (TT.supportsCOMDAT()) {
1221 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
1222 ProfileNameVar->setComdat(M.getOrInsertComdat(
1223 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename")));
1224 }
1225}
1226
1227Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1228 const std::string &TestFilename,
1229 bool IsCS) {
1230 auto getProfileSum = [IsCS](const std::string &Filename,
1231 CountSumOrPercent &Sum) -> Error {
1232 auto ReaderOrErr = InstrProfReader::create(Filename);
1233 if (Error E = ReaderOrErr.takeError()) {
1234 return E;
1235 }
1236 auto Reader = std::move(ReaderOrErr.get());
1237 Reader->accumulateCounts(Sum, IsCS);
1238 return Error::success();
1239 };
1240 auto Ret = getProfileSum(BaseFilename, Base);
1241 if (Ret)
1242 return Ret;
1243 Ret = getProfileSum(TestFilename, Test);
1244 if (Ret)
1245 return Ret;
1246 this->BaseFilename = &BaseFilename;
1247 this->TestFilename = &TestFilename;
1248 Valid = true;
1249 return Error::success();
1250}
1251
1252void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) {
1253 Mismatch.NumEntries += 1;
1254 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1255 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1256 if (Test.ValueCounts[I] >= 1.0f)
1257 Mismatch.ValueCounts[I] +=
1258 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1259 }
1260}
1261
1262void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) {
1263 Unique.NumEntries += 1;
1264 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1265 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1266 if (Test.ValueCounts[I] >= 1.0f)
1267 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1268 }
1269}
1270
1271void OverlapStats::dump(raw_fd_ostream &OS) const {
1272 if (!Valid)
1273 return;
1274
1275 const char *EntryName =
1276 (Level == ProgramLevel ? "functions" : "edge counters");
1277 if (Level == ProgramLevel) {
1278 OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1279 << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1280 } else {
1281 OS << "Function level:\n"
1282 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1283 }
1284
1285 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1286 if (Mismatch.NumEntries)
1287 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1288 << "\n";
1289 if (Unique.NumEntries)
1290 OS << " # of " << EntryName
1291 << " only in test_profile: " << Unique.NumEntries << "\n";
1292
1293 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1294 << "\n";
1295 if (Mismatch.NumEntries)
1296 OS << " Mismatched count percentage (Edge): "
1297 << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1298 if (Unique.NumEntries)
1299 OS << " Percentage of Edge profile only in test_profile: "
1300 << format("%.3f%%", Unique.CountSum * 100) << "\n";
1301 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1302 << "\n"
1303 << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1304 << "\n";
1305
1306 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1307 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1308 continue;
1309 char ProfileKindName[20];
1310 switch (I) {
1311 case IPVK_IndirectCallTarget:
1312 strncpy(ProfileKindName, "IndirectCall", 19);
1313 break;
1314 case IPVK_MemOPSize:
1315 strncpy(ProfileKindName, "MemOP", 19);
1316 break;
1317 default:
1318 snprintf(ProfileKindName, 19, "VP[%d]", I);
1319 break;
1320 }
1321 OS << " " << ProfileKindName
1322 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1323 << "\n";
1324 if (Mismatch.NumEntries)
1325 OS << " Mismatched count percentage (" << ProfileKindName
1326 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1327 if (Unique.NumEntries)
1328 OS << " Percentage of " << ProfileKindName
1329 << " profile only in test_profile: "
1330 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1331 OS << " " << ProfileKindName
1332 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1333 << "\n"
1334 << " " << ProfileKindName
1335 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1336 << "\n";
1337 }
1338}
1339
1340} // end namespace llvm

/build/llvm-toolchain-snapshot-14~++20220127100629+cd20e579df07/llvm/include/llvm/Support/LEB128.h

1//===- llvm/Support/LEB128.h - [SU]LEB128 utility functions -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares some utility functions for encoding SLEB128 and
10// ULEB128 values.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_LEB128_H
15#define LLVM_SUPPORT_LEB128_H
16
17#include "llvm/Support/raw_ostream.h"
18
19namespace llvm {
20
21/// Utility function to encode a SLEB128 value to an output stream. Returns
22/// the length in bytes of the encoded value.
23inline unsigned encodeSLEB128(int64_t Value, raw_ostream &OS,
24 unsigned PadTo = 0) {
25 bool More;
26 unsigned Count = 0;
27 do {
28 uint8_t Byte = Value & 0x7f;
29 // NOTE: this assumes that this signed shift is an arithmetic right shift.
30 Value >>= 7;
31 More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) ||
32 ((Value == -1) && ((Byte & 0x40) != 0))));
33 Count++;
34 if (More || Count < PadTo)
35 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
36 OS << char(Byte);
37 } while (More);
38
39 // Pad with 0x80 and emit a terminating byte at the end.
40 if (Count < PadTo) {
41 uint8_t PadValue = Value < 0 ? 0x7f : 0x00;
42 for (; Count < PadTo - 1; ++Count)
43 OS << char(PadValue | 0x80);
44 OS << char(PadValue);
45 Count++;
46 }
47 return Count;
48}
49
50/// Utility function to encode a SLEB128 value to a buffer. Returns
51/// the length in bytes of the encoded value.
52inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned PadTo = 0) {
53 uint8_t *orig_p = p;
54 unsigned Count = 0;
55 bool More;
56 do {
57 uint8_t Byte = Value & 0x7f;
58 // NOTE: this assumes that this signed shift is an arithmetic right shift.
59 Value >>= 7;
60 More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) ||
61 ((Value == -1) && ((Byte & 0x40) != 0))));
62 Count++;
63 if (More || Count < PadTo)
64 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
65 *p++ = Byte;
66 } while (More);
67
68 // Pad with 0x80 and emit a terminating byte at the end.
69 if (Count < PadTo) {
70 uint8_t PadValue = Value < 0 ? 0x7f : 0x00;
71 for (; Count < PadTo - 1; ++Count)
72 *p++ = (PadValue | 0x80);
73 *p++ = PadValue;
74 }
75 return (unsigned)(p - orig_p);
76}
77
78/// Utility function to encode a ULEB128 value to an output stream. Returns
79/// the length in bytes of the encoded value.
80inline unsigned encodeULEB128(uint64_t Value, raw_ostream &OS,
81 unsigned PadTo = 0) {
82 unsigned Count = 0;
83 do {
84 uint8_t Byte = Value & 0x7f;
85 Value >>= 7;
86 Count++;
87 if (Value != 0 || Count < PadTo)
88 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
89 OS << char(Byte);
90 } while (Value != 0);
91
92 // Pad with 0x80 and emit a null byte at the end.
93 if (Count < PadTo) {
94 for (; Count < PadTo - 1; ++Count)
95 OS << '\x80';
96 OS << '\x00';
97 Count++;
98 }
99 return Count;
100}
101
102/// Utility function to encode a ULEB128 value to a buffer. Returns
103/// the length in bytes of the encoded value.
104inline unsigned encodeULEB128(uint64_t Value, uint8_t *p,
105 unsigned PadTo = 0) {
106 uint8_t *orig_p = p;
107 unsigned Count = 0;
108 do {
109 uint8_t Byte = Value & 0x7f;
110 Value >>= 7;
111 Count++;
112 if (Value != 0 || Count < PadTo)
113 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
114 *p++ = Byte;
115 } while (Value != 0);
116
117 // Pad with 0x80 and emit a null byte at the end.
118 if (Count < PadTo) {
119 for (; Count < PadTo - 1; ++Count)
120 *p++ = '\x80';
121 *p++ = '\x00';
122 }
123
124 return (unsigned)(p - orig_p);
125}
126
127/// Utility function to decode a ULEB128 value.
128inline uint64_t decodeULEB128(const uint8_t *p, unsigned *n = nullptr,
129 const uint8_t *end = nullptr,
130 const char **error = nullptr) {
131 const uint8_t *orig_p = p;
132 uint64_t Value = 0;
133 unsigned Shift = 0;
134 if (error
3.1
'error' is null
3.1
'error' is null
)
4
Taking false branch
135 *error = nullptr;
136 do {
137 if (p == end) {
5
Assuming 'p' is equal to 'end'
6
Taking true branch
138 if (error
6.1
'error' is null
6.1
'error' is null
)
7
Taking false branch
139 *error = "malformed uleb128, extends past end";
140 if (n
7.1
'n' is non-null
7.1
'n' is non-null
)
8
Taking true branch
141 *n = (unsigned)(p - orig_p);
142 return 0;
9
Returning zero
143 }
144 uint64_t Slice = *p & 0x7f;
145 if ((Shift >= 64 && Slice != 0) || Slice << Shift >> Shift != Slice) {
146 if (error)
147 *error = "uleb128 too big for uint64";
148 if (n)
149 *n = (unsigned)(p - orig_p);
150 return 0;
151 }
152 Value += Slice << Shift;
153 Shift += 7;
154 } while (*p++ >= 128);
155 if (n)
156 *n = (unsigned)(p - orig_p);
157 return Value;
158}
159
160/// Utility function to decode a SLEB128 value.
161inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr,
162 const uint8_t *end = nullptr,
163 const char **error = nullptr) {
164 const uint8_t *orig_p = p;
165 int64_t Value = 0;
166 unsigned Shift = 0;
167 uint8_t Byte;
168 if (error)
169 *error = nullptr;
170 do {
171 if (p == end) {
172 if (error)
173 *error = "malformed sleb128, extends past end";
174 if (n)
175 *n = (unsigned)(p - orig_p);
176 return 0;
177 }
178 Byte = *p;
179 uint64_t Slice = Byte & 0x7f;
180 if ((Shift >= 64 && Slice != (Value < 0 ? 0x7f : 0x00)) ||
181 (Shift == 63 && Slice != 0 && Slice != 0x7f)) {
182 if (error)
183 *error = "sleb128 too big for int64";
184 if (n)
185 *n = (unsigned)(p - orig_p);
186 return 0;
187 }
188 Value |= Slice << Shift;
189 Shift += 7;
190 ++p;
191 } while (Byte >= 128);
192 // Sign extend negative numbers if needed.
193 if (Shift < 64 && (Byte & 0x40))
194 Value |= (-1ULL) << Shift;
195 if (n)
196 *n = (unsigned)(p - orig_p);
197 return Value;
198}
199
200/// Utility function to get the size of the ULEB128-encoded value.
201extern unsigned getULEB128Size(uint64_t Value);
202
203/// Utility function to get the size of the SLEB128-encoded value.
204extern unsigned getSLEB128Size(int64_t Value);
205
206} // namespace llvm
207
208#endif // LLVM_SUPPORT_LEB128_H