Bug Summary

File:llvm/lib/ProfileData/InstrProf.cpp
Warning:line 497, column 24
Dereference of null pointer (loaded from variable 'P')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name InstrProf.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/ProfileData -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/ProfileData -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/ProfileData -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-04-14-063029-18377-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp

/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp

1//===- InstrProf.cpp - Instrumented profiling format support --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for clang's instrumentation based PGO and
10// coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProf.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallString.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Triple.h"
21#include "llvm/Config/config.h"
22#include "llvm/IR/Constant.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalValue.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/Instruction.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/IR/Metadata.h"
31#include "llvm/IR/Module.h"
32#include "llvm/IR/Type.h"
33#include "llvm/ProfileData/InstrProfReader.h"
34#include "llvm/Support/Casting.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/Compiler.h"
37#include "llvm/Support/Compression.h"
38#include "llvm/Support/Endian.h"
39#include "llvm/Support/Error.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/LEB128.h"
42#include "llvm/Support/ManagedStatic.h"
43#include "llvm/Support/MathExtras.h"
44#include "llvm/Support/Path.h"
45#include "llvm/Support/SwapByteOrder.h"
46#include <algorithm>
47#include <cassert>
48#include <cstddef>
49#include <cstdint>
50#include <cstring>
51#include <memory>
52#include <string>
53#include <system_error>
54#include <utility>
55#include <vector>
56
57using namespace llvm;
58
59static cl::opt<bool> StaticFuncFullModulePrefix(
60 "static-func-full-module-prefix", cl::init(true), cl::Hidden,
61 cl::desc("Use full module build paths in the profile counter names for "
62 "static functions."));
63
64// This option is tailored to users that have different top-level directory in
65// profile-gen and profile-use compilation. Users need to specific the number
66// of levels to strip. A value larger than the number of directories in the
67// source file will strip all the directory names and only leave the basename.
68//
69// Note current ThinLTO module importing for the indirect-calls assumes
70// the source directory name not being stripped. A non-zero option value here
71// can potentially prevent some inter-module indirect-call-promotions.
72static cl::opt<unsigned> StaticFuncStripDirNamePrefix(
73 "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
74 cl::desc("Strip specified level of directory name from source path in "
75 "the profile counter name for static functions."));
76
77static std::string getInstrProfErrString(instrprof_error Err) {
78 switch (Err) {
79 case instrprof_error::success:
80 return "Success";
81 case instrprof_error::eof:
82 return "End of File";
83 case instrprof_error::unrecognized_format:
84 return "Unrecognized instrumentation profile encoding format";
85 case instrprof_error::bad_magic:
86 return "Invalid instrumentation profile data (bad magic)";
87 case instrprof_error::bad_header:
88 return "Invalid instrumentation profile data (file header is corrupt)";
89 case instrprof_error::unsupported_version:
90 return "Unsupported instrumentation profile format version";
91 case instrprof_error::unsupported_hash_type:
92 return "Unsupported instrumentation profile hash type";
93 case instrprof_error::too_large:
94 return "Too much profile data";
95 case instrprof_error::truncated:
96 return "Truncated profile data";
97 case instrprof_error::malformed:
98 return "Malformed instrumentation profile data";
99 case instrprof_error::invalid_prof:
100 return "Invalid profile created. Please file a bug "
101 "at: " BUG_REPORT_URL"https://bugs.llvm.org/"
102 " and include the profraw files that caused this error.";
103 case instrprof_error::unknown_function:
104 return "No profile data available for function";
105 case instrprof_error::hash_mismatch:
106 return "Function control flow change detected (hash mismatch)";
107 case instrprof_error::count_mismatch:
108 return "Function basic block count change detected (counter mismatch)";
109 case instrprof_error::counter_overflow:
110 return "Counter overflow";
111 case instrprof_error::value_site_count_mismatch:
112 return "Function value site count change detected (counter mismatch)";
113 case instrprof_error::compress_failed:
114 return "Failed to compress data (zlib)";
115 case instrprof_error::uncompress_failed:
116 return "Failed to uncompress data (zlib)";
117 case instrprof_error::empty_raw_profile:
118 return "Empty raw profile file";
119 case instrprof_error::zlib_unavailable:
120 return "Profile uses zlib compression but the profile reader was built without zlib support";
121 }
122 llvm_unreachable("A value of instrprof_error has no message.")::llvm::llvm_unreachable_internal("A value of instrprof_error has no message."
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 122)
;
123}
124
125namespace {
126
127// FIXME: This class is only here to support the transition to llvm::Error. It
128// will be removed once this transition is complete. Clients should prefer to
129// deal with the Error value directly, rather than converting to error_code.
130class InstrProfErrorCategoryType : public std::error_category {
131 const char *name() const noexcept override { return "llvm.instrprof"; }
132
133 std::string message(int IE) const override {
134 return getInstrProfErrString(static_cast<instrprof_error>(IE));
135 }
136};
137
138} // end anonymous namespace
139
140static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
141
142const std::error_category &llvm::instrprof_category() {
143 return *ErrorCategory;
144}
145
146namespace {
147
148const char *InstrProfSectNameCommon[] = {
149#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
150 SectNameCommon,
151#include "llvm/ProfileData/InstrProfData.inc"
152};
153
154const char *InstrProfSectNameCoff[] = {
155#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
156 SectNameCoff,
157#include "llvm/ProfileData/InstrProfData.inc"
158};
159
160const char *InstrProfSectNamePrefix[] = {
161#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
162 Prefix,
163#include "llvm/ProfileData/InstrProfData.inc"
164};
165
166} // namespace
167
168namespace llvm {
169
170cl::opt<bool> DoInstrProfNameCompression(
171 "enable-name-compression",
172 cl::desc("Enable name/filename string compression"), cl::init(true));
173
174std::string getInstrProfSectionName(InstrProfSectKind IPSK,
175 Triple::ObjectFormatType OF,
176 bool AddSegmentInfo) {
177 std::string SectName;
178
179 if (OF == Triple::MachO && AddSegmentInfo)
180 SectName = InstrProfSectNamePrefix[IPSK];
181
182 if (OF == Triple::COFF)
183 SectName += InstrProfSectNameCoff[IPSK];
184 else
185 SectName += InstrProfSectNameCommon[IPSK];
186
187 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
188 SectName += ",regular,live_support";
189
190 return SectName;
191}
192
193void SoftInstrProfErrors::addError(instrprof_error IE) {
194 if (IE == instrprof_error::success)
195 return;
196
197 if (FirstError == instrprof_error::success)
198 FirstError = IE;
199
200 switch (IE) {
201 case instrprof_error::hash_mismatch:
202 ++NumHashMismatches;
203 break;
204 case instrprof_error::count_mismatch:
205 ++NumCountMismatches;
206 break;
207 case instrprof_error::counter_overflow:
208 ++NumCounterOverflows;
209 break;
210 case instrprof_error::value_site_count_mismatch:
211 ++NumValueSiteCountMismatches;
212 break;
213 default:
214 llvm_unreachable("Not a soft error")::llvm::llvm_unreachable_internal("Not a soft error", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 214)
;
215 }
216}
217
218std::string InstrProfError::message() const {
219 return getInstrProfErrString(Err);
220}
221
222char InstrProfError::ID = 0;
223
224std::string getPGOFuncName(StringRef RawFuncName,
225 GlobalValue::LinkageTypes Linkage,
226 StringRef FileName,
227 uint64_t Version LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__))) {
228 return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
229}
230
231// Strip NumPrefix level of directory name from PathNameStr. If the number of
232// directory separators is less than NumPrefix, strip all the directories and
233// leave base file name only.
234static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
235 uint32_t Count = NumPrefix;
236 uint32_t Pos = 0, LastPos = 0;
237 for (auto & CI : PathNameStr) {
238 ++Pos;
239 if (llvm::sys::path::is_separator(CI)) {
240 LastPos = Pos;
241 --Count;
242 }
243 if (Count == 0)
244 break;
245 }
246 return PathNameStr.substr(LastPos);
247}
248
249// Return the PGOFuncName. This function has some special handling when called
250// in LTO optimization. The following only applies when calling in LTO passes
251// (when \c InLTO is true): LTO's internalization privatizes many global linkage
252// symbols. This happens after value profile annotation, but those internal
253// linkage functions should not have a source prefix.
254// Additionally, for ThinLTO mode, exported internal functions are promoted
255// and renamed. We need to ensure that the original internal PGO name is
256// used when computing the GUID that is compared against the profiled GUIDs.
257// To differentiate compiler generated internal symbols from original ones,
258// PGOFuncName meta data are created and attached to the original internal
259// symbols in the value profile annotation step
260// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
261// data, its original linkage must be non-internal.
262std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
263 if (!InLTO) {
264 StringRef FileName(F.getParent()->getSourceFileName());
265 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
266 if (StripLevel < StaticFuncStripDirNamePrefix)
267 StripLevel = StaticFuncStripDirNamePrefix;
268 if (StripLevel)
269 FileName = stripDirPrefix(FileName, StripLevel);
270 return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
271 }
272
273 // In LTO mode (when InLTO is true), first check if there is a meta data.
274 if (MDNode *MD = getPGOFuncNameMetadata(F)) {
275 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
276 return S.str();
277 }
278
279 // If there is no meta data, the function must be a global before the value
280 // profile annotation pass. Its current linkage may be internal if it is
281 // internalized in LTO mode.
282 return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
283}
284
285StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
286 if (FileName.empty())
287 return PGOFuncName;
288 // Drop the file name including ':'. See also getPGOFuncName.
289 if (PGOFuncName.startswith(FileName))
290 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
291 return PGOFuncName;
292}
293
294// \p FuncName is the string used as profile lookup key for the function. A
295// symbol is created to hold the name. Return the legalized symbol name.
296std::string getPGOFuncNameVarName(StringRef FuncName,
297 GlobalValue::LinkageTypes Linkage) {
298 std::string VarName = std::string(getInstrProfNameVarPrefix());
299 VarName += FuncName;
300
301 if (!GlobalValue::isLocalLinkage(Linkage))
302 return VarName;
303
304 // Now fix up illegal chars in local VarName that may upset the assembler.
305 const char *InvalidChars = "-:<>/\"'";
306 size_t found = VarName.find_first_of(InvalidChars);
307 while (found != std::string::npos) {
308 VarName[found] = '_';
309 found = VarName.find_first_of(InvalidChars, found + 1);
310 }
311 return VarName;
312}
313
314GlobalVariable *createPGOFuncNameVar(Module &M,
315 GlobalValue::LinkageTypes Linkage,
316 StringRef PGOFuncName) {
317 // We generally want to match the function's linkage, but available_externally
318 // and extern_weak both have the wrong semantics, and anything that doesn't
319 // need to link across compilation units doesn't need to be visible at all.
320 if (Linkage == GlobalValue::ExternalWeakLinkage)
321 Linkage = GlobalValue::LinkOnceAnyLinkage;
322 else if (Linkage == GlobalValue::AvailableExternallyLinkage)
323 Linkage = GlobalValue::LinkOnceODRLinkage;
324 else if (Linkage == GlobalValue::InternalLinkage ||
325 Linkage == GlobalValue::ExternalLinkage)
326 Linkage = GlobalValue::PrivateLinkage;
327
328 auto *Value =
329 ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
330 auto FuncNameVar =
331 new GlobalVariable(M, Value->getType(), true, Linkage, Value,
332 getPGOFuncNameVarName(PGOFuncName, Linkage));
333
334 // Hide the symbol so that we correctly get a copy for each executable.
335 if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
336 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
337
338 return FuncNameVar;
339}
340
341GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
342 return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
343}
344
345Error InstrProfSymtab::create(Module &M, bool InLTO) {
346 for (Function &F : M) {
347 // Function may not have a name: like using asm("") to overwrite the name.
348 // Ignore in this case.
349 if (!F.hasName())
350 continue;
351 const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
352 if (Error E = addFuncName(PGOFuncName))
353 return E;
354 MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
355 // In ThinLTO, local function may have been promoted to global and have
356 // suffix ".llvm." added to the function name. We need to add the
357 // stripped function name to the symbol table so that we can find a match
358 // from profile.
359 //
360 // We may have other suffixes similar as ".llvm." which are needed to
361 // be stripped before the matching, but ".__uniq." suffix which is used
362 // to differentiate internal linkage functions in different modules
363 // should be kept. Now this is the only suffix with the pattern ".xxx"
364 // which is kept before matching.
365 const std::string UniqSuffix = ".__uniq.";
366 auto pos = PGOFuncName.find(UniqSuffix);
367 // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
368 // search '.' from the beginning.
369 if (pos != std::string::npos)
370 pos += UniqSuffix.length();
371 else
372 pos = 0;
373 pos = PGOFuncName.find('.', pos);
374 if (pos != std::string::npos && pos != 0) {
375 const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
376 if (Error E = addFuncName(OtherFuncName))
377 return E;
378 MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
379 }
380 }
381 Sorted = false;
382 finalizeSymtab();
383 return Error::success();
384}
385
386uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
387 finalizeSymtab();
388 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
389 return A.first < Address;
390 });
391 // Raw function pointer collected by value profiler may be from
392 // external functions that are not instrumented. They won't have
393 // mapping data to be used by the deserializer. Force the value to
394 // be 0 in this case.
395 if (It != AddrToMD5Map.end() && It->first == Address)
396 return (uint64_t)It->second;
397 return 0;
398}
399
400Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
401 bool doCompression, std::string &Result) {
402 assert(!NameStrs.empty() && "No name data to emit")((!NameStrs.empty() && "No name data to emit") ? static_cast
<void> (0) : __assert_fail ("!NameStrs.empty() && \"No name data to emit\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 402, __PRETTY_FUNCTION__))
;
403
404 uint8_t Header[16], *P = Header;
405 std::string UncompressedNameStrings =
406 join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
407
408 assert(StringRef(UncompressedNameStrings)((StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator
()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)"
) ? static_cast<void> (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 410, __PRETTY_FUNCTION__))
409 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&((StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator
()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)"
) ? static_cast<void> (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 410, __PRETTY_FUNCTION__))
410 "PGO name is invalid (contains separator token)")((StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator
()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)"
) ? static_cast<void> (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 410, __PRETTY_FUNCTION__))
;
411
412 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
413 P += EncLen;
414
415 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
416 EncLen = encodeULEB128(CompressedLen, P);
417 P += EncLen;
418 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
419 unsigned HeaderLen = P - &Header[0];
420 Result.append(HeaderStr, HeaderLen);
421 Result += InputStr;
422 return Error::success();
423 };
424
425 if (!doCompression) {
426 return WriteStringToResult(0, UncompressedNameStrings);
427 }
428
429 SmallString<128> CompressedNameStrings;
430 Error E = zlib::compress(StringRef(UncompressedNameStrings),
431 CompressedNameStrings, zlib::BestSizeCompression);
432 if (E) {
433 consumeError(std::move(E));
434 return make_error<InstrProfError>(instrprof_error::compress_failed);
435 }
436
437 return WriteStringToResult(CompressedNameStrings.size(),
438 CompressedNameStrings);
439}
440
441StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
442 auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
443 StringRef NameStr =
444 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
445 return NameStr;
446}
447
448Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
449 std::string &Result, bool doCompression) {
450 std::vector<std::string> NameStrs;
451 for (auto *NameVar : NameVars) {
452 NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
453 }
454 return collectPGOFuncNameStrings(
455 NameStrs, zlib::isAvailable() && doCompression, Result);
456}
457
458Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
459 const uint8_t *P = NameStrings.bytes_begin();
460 const uint8_t *EndP = NameStrings.bytes_end();
461 while (P < EndP) {
1
Assuming 'P' is < 'EndP'
2
Loop condition is true. Entering loop body
462 uint32_t N;
463 uint64_t UncompressedSize = decodeULEB128(P, &N);
3
Calling 'decodeULEB128'
10
Returning from 'decodeULEB128'
11
'UncompressedSize' initialized to 0
464 P += N;
465 uint64_t CompressedSize = decodeULEB128(P, &N);
466 P += N;
467 bool isCompressed = (CompressedSize != 0);
468 SmallString<128> UncompressedNameStrings;
469 StringRef NameStrings;
470 if (isCompressed
11.1
'isCompressed' is false
11.1
'isCompressed' is false
) {
12
Taking false branch
471 if (!llvm::zlib::isAvailable())
472 return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
473
474 StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
475 CompressedSize);
476 if (Error E =
477 zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
478 UncompressedSize)) {
479 consumeError(std::move(E));
480 return make_error<InstrProfError>(instrprof_error::uncompress_failed);
481 }
482 P += CompressedSize;
483 NameStrings = StringRef(UncompressedNameStrings.data(),
484 UncompressedNameStrings.size());
485 } else {
486 NameStrings =
487 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
488 P += UncompressedSize;
13
Null pointer value stored to 'P'
489 }
490 // Now parse the name strings.
491 SmallVector<StringRef, 0> Names;
492 NameStrings.split(Names, getInstrProfNameSeparator());
493 for (StringRef &Name : Names)
14
Assuming '__begin2' is equal to '__end2'
494 if (Error E = Symtab.addFuncName(Name))
495 return E;
496
497 while (P
14.1
'P' is < 'EndP'
14.1
'P' is < 'EndP'
< EndP && *P == 0)
15
Dereference of null pointer (loaded from variable 'P')
498 P++;
499 }
500 return Error::success();
501}
502
503void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
504 uint64_t FuncSum = 0;
505 Sum.NumEntries += Counts.size();
506 for (size_t F = 0, E = Counts.size(); F < E; ++F)
507 FuncSum += Counts[F];
508 Sum.CountSum += FuncSum;
509
510 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
511 uint64_t KindSum = 0;
512 uint32_t NumValueSites = getNumValueSites(VK);
513 for (size_t I = 0; I < NumValueSites; ++I) {
514 uint32_t NV = getNumValueDataForSite(VK, I);
515 std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I);
516 for (uint32_t V = 0; V < NV; V++)
517 KindSum += VD[V].Count;
518 }
519 Sum.ValueCounts[VK] += KindSum;
520 }
521}
522
523void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input,
524 uint32_t ValueKind,
525 OverlapStats &Overlap,
526 OverlapStats &FuncLevelOverlap) {
527 this->sortByTargetValues();
528 Input.sortByTargetValues();
529 double Score = 0.0f, FuncLevelScore = 0.0f;
530 auto I = ValueData.begin();
531 auto IE = ValueData.end();
532 auto J = Input.ValueData.begin();
533 auto JE = Input.ValueData.end();
534 while (I != IE && J != JE) {
535 if (I->Value == J->Value) {
536 Score += OverlapStats::score(I->Count, J->Count,
537 Overlap.Base.ValueCounts[ValueKind],
538 Overlap.Test.ValueCounts[ValueKind]);
539 FuncLevelScore += OverlapStats::score(
540 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
541 FuncLevelOverlap.Test.ValueCounts[ValueKind]);
542 ++I;
543 } else if (I->Value < J->Value) {
544 ++I;
545 continue;
546 }
547 ++J;
548 }
549 Overlap.Overlap.ValueCounts[ValueKind] += Score;
550 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
551}
552
553// Return false on mismatch.
554void InstrProfRecord::overlapValueProfData(uint32_t ValueKind,
555 InstrProfRecord &Other,
556 OverlapStats &Overlap,
557 OverlapStats &FuncLevelOverlap) {
558 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
559 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind))((ThisNumValueSites == Other.getNumValueSites(ValueKind)) ? static_cast
<void> (0) : __assert_fail ("ThisNumValueSites == Other.getNumValueSites(ValueKind)"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 559, __PRETTY_FUNCTION__))
;
560 if (!ThisNumValueSites)
561 return;
562
563 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
564 getOrCreateValueSitesForKind(ValueKind);
565 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
566 Other.getValueSitesForKind(ValueKind);
567 for (uint32_t I = 0; I < ThisNumValueSites; I++)
568 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
569 FuncLevelOverlap);
570}
571
572void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap,
573 OverlapStats &FuncLevelOverlap,
574 uint64_t ValueCutoff) {
575 // FuncLevel CountSum for other should already computed and nonzero.
576 assert(FuncLevelOverlap.Test.CountSum >= 1.0f)((FuncLevelOverlap.Test.CountSum >= 1.0f) ? static_cast<
void> (0) : __assert_fail ("FuncLevelOverlap.Test.CountSum >= 1.0f"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 576, __PRETTY_FUNCTION__))
;
577 accumulateCounts(FuncLevelOverlap.Base);
578 bool Mismatch = (Counts.size() != Other.Counts.size());
579
580 // Check if the value profiles mismatch.
581 if (!Mismatch) {
582 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
583 uint32_t ThisNumValueSites = getNumValueSites(Kind);
584 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
585 if (ThisNumValueSites != OtherNumValueSites) {
586 Mismatch = true;
587 break;
588 }
589 }
590 }
591 if (Mismatch) {
592 Overlap.addOneMismatch(FuncLevelOverlap.Test);
593 return;
594 }
595
596 // Compute overlap for value counts.
597 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
598 overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
599
600 double Score = 0.0;
601 uint64_t MaxCount = 0;
602 // Compute overlap for edge counts.
603 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
604 Score += OverlapStats::score(Counts[I], Other.Counts[I],
605 Overlap.Base.CountSum, Overlap.Test.CountSum);
606 MaxCount = std::max(Other.Counts[I], MaxCount);
607 }
608 Overlap.Overlap.CountSum += Score;
609 Overlap.Overlap.NumEntries += 1;
610
611 if (MaxCount >= ValueCutoff) {
612 double FuncScore = 0.0;
613 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
614 FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
615 FuncLevelOverlap.Base.CountSum,
616 FuncLevelOverlap.Test.CountSum);
617 FuncLevelOverlap.Overlap.CountSum = FuncScore;
618 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
619 FuncLevelOverlap.Valid = true;
620 }
621}
622
623void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
624 uint64_t Weight,
625 function_ref<void(instrprof_error)> Warn) {
626 this->sortByTargetValues();
627 Input.sortByTargetValues();
628 auto I = ValueData.begin();
629 auto IE = ValueData.end();
630 for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
631 ++J) {
632 while (I != IE && I->Value < J->Value)
633 ++I;
634 if (I != IE && I->Value == J->Value) {
635 bool Overflowed;
636 I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed);
637 if (Overflowed)
638 Warn(instrprof_error::counter_overflow);
639 ++I;
640 continue;
641 }
642 ValueData.insert(I, *J);
643 }
644}
645
646void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D,
647 function_ref<void(instrprof_error)> Warn) {
648 for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
649 bool Overflowed;
650 I->Count = SaturatingMultiply(I->Count, N, &Overflowed) / D;
651 if (Overflowed)
652 Warn(instrprof_error::counter_overflow);
653 }
654}
655
656// Merge Value Profile data from Src record to this record for ValueKind.
657// Scale merged value counts by \p Weight.
658void InstrProfRecord::mergeValueProfData(
659 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
660 function_ref<void(instrprof_error)> Warn) {
661 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
662 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
663 if (ThisNumValueSites != OtherNumValueSites) {
664 Warn(instrprof_error::value_site_count_mismatch);
665 return;
666 }
667 if (!ThisNumValueSites)
668 return;
669 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
670 getOrCreateValueSitesForKind(ValueKind);
671 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
672 Src.getValueSitesForKind(ValueKind);
673 for (uint32_t I = 0; I < ThisNumValueSites; I++)
674 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
675}
676
677void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
678 function_ref<void(instrprof_error)> Warn) {
679 // If the number of counters doesn't match we either have bad data
680 // or a hash collision.
681 if (Counts.size() != Other.Counts.size()) {
682 Warn(instrprof_error::count_mismatch);
683 return;
684 }
685
686 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
687 bool Overflowed;
688 Counts[I] =
689 SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
690 if (Overflowed)
691 Warn(instrprof_error::counter_overflow);
692 }
693
694 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
695 mergeValueProfData(Kind, Other, Weight, Warn);
696}
697
698void InstrProfRecord::scaleValueProfData(
699 uint32_t ValueKind, uint64_t N, uint64_t D,
700 function_ref<void(instrprof_error)> Warn) {
701 for (auto &R : getValueSitesForKind(ValueKind))
702 R.scale(N, D, Warn);
703}
704
705void InstrProfRecord::scale(uint64_t N, uint64_t D,
706 function_ref<void(instrprof_error)> Warn) {
707 assert(D != 0 && "D cannot be 0")((D != 0 && "D cannot be 0") ? static_cast<void>
(0) : __assert_fail ("D != 0 && \"D cannot be 0\"", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 707, __PRETTY_FUNCTION__))
;
708 for (auto &Count : this->Counts) {
709 bool Overflowed;
710 Count = SaturatingMultiply(Count, N, &Overflowed) / D;
711 if (Overflowed)
712 Warn(instrprof_error::counter_overflow);
713 }
714 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
715 scaleValueProfData(Kind, N, D, Warn);
716}
717
718// Map indirect call target name hash to name string.
719uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
720 InstrProfSymtab *SymTab) {
721 if (!SymTab)
722 return Value;
723
724 if (ValueKind == IPVK_IndirectCallTarget)
725 return SymTab->getFunctionHashFromAddress(Value);
726
727 return Value;
728}
729
730void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
731 InstrProfValueData *VData, uint32_t N,
732 InstrProfSymtab *ValueMap) {
733 for (uint32_t I = 0; I < N; I++) {
734 VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
735 }
736 std::vector<InstrProfValueSiteRecord> &ValueSites =
737 getOrCreateValueSitesForKind(ValueKind);
738 if (N == 0)
739 ValueSites.emplace_back();
740 else
741 ValueSites.emplace_back(VData, VData + N);
742}
743
744#define INSTR_PROF_COMMON_API_IMPL
745#include "llvm/ProfileData/InstrProfData.inc"
746
747/*!
748 * ValueProfRecordClosure Interface implementation for InstrProfRecord
749 * class. These C wrappers are used as adaptors so that C++ code can be
750 * invoked as callbacks.
751 */
752uint32_t getNumValueKindsInstrProf(const void *Record) {
753 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
754}
755
756uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
757 return reinterpret_cast<const InstrProfRecord *>(Record)
758 ->getNumValueSites(VKind);
759}
760
761uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
762 return reinterpret_cast<const InstrProfRecord *>(Record)
763 ->getNumValueData(VKind);
764}
765
766uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
767 uint32_t S) {
768 return reinterpret_cast<const InstrProfRecord *>(R)
769 ->getNumValueDataForSite(VK, S);
770}
771
772void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
773 uint32_t K, uint32_t S) {
774 reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
775}
776
777ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
778 ValueProfData *VD =
779 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
780 memset(VD, 0, TotalSizeInBytes);
781 return VD;
782}
783
784static ValueProfRecordClosure InstrProfRecordClosure = {
785 nullptr,
786 getNumValueKindsInstrProf,
787 getNumValueSitesInstrProf,
788 getNumValueDataInstrProf,
789 getNumValueDataForSiteInstrProf,
790 nullptr,
791 getValueForSiteInstrProf,
792 allocValueProfDataInstrProf};
793
794// Wrapper implementation using the closure mechanism.
795uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
796 auto Closure = InstrProfRecordClosure;
797 Closure.Record = &Record;
798 return getValueProfDataSize(&Closure);
799}
800
801// Wrapper implementation using the closure mechanism.
802std::unique_ptr<ValueProfData>
803ValueProfData::serializeFrom(const InstrProfRecord &Record) {
804 InstrProfRecordClosure.Record = &Record;
805
806 std::unique_ptr<ValueProfData> VPD(
807 serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
808 return VPD;
809}
810
811void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
812 InstrProfSymtab *SymTab) {
813 Record.reserveSites(Kind, NumValueSites);
814
815 InstrProfValueData *ValueData = getValueProfRecordValueData(this);
816 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
817 uint8_t ValueDataCount = this->SiteCountArray[VSite];
818 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
819 ValueData += ValueDataCount;
820 }
821}
822
823// For writing/serializing, Old is the host endianness, and New is
824// byte order intended on disk. For Reading/deserialization, Old
825// is the on-disk source endianness, and New is the host endianness.
826void ValueProfRecord::swapBytes(support::endianness Old,
827 support::endianness New) {
828 using namespace support;
829
830 if (Old == New)
831 return;
832
833 if (getHostEndianness() != Old) {
834 sys::swapByteOrder<uint32_t>(NumValueSites);
835 sys::swapByteOrder<uint32_t>(Kind);
836 }
837 uint32_t ND = getValueProfRecordNumValueData(this);
838 InstrProfValueData *VD = getValueProfRecordValueData(this);
839
840 // No need to swap byte array: SiteCountArrray.
841 for (uint32_t I = 0; I < ND; I++) {
842 sys::swapByteOrder<uint64_t>(VD[I].Value);
843 sys::swapByteOrder<uint64_t>(VD[I].Count);
844 }
845 if (getHostEndianness() == Old) {
846 sys::swapByteOrder<uint32_t>(NumValueSites);
847 sys::swapByteOrder<uint32_t>(Kind);
848 }
849}
850
851void ValueProfData::deserializeTo(InstrProfRecord &Record,
852 InstrProfSymtab *SymTab) {
853 if (NumValueKinds == 0)
854 return;
855
856 ValueProfRecord *VR = getFirstValueProfRecord(this);
857 for (uint32_t K = 0; K < NumValueKinds; K++) {
858 VR->deserializeTo(Record, SymTab);
859 VR = getValueProfRecordNext(VR);
860 }
861}
862
863template <class T>
864static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
865 using namespace support;
866
867 if (Orig == little)
868 return endian::readNext<T, little, unaligned>(D);
869 else
870 return endian::readNext<T, big, unaligned>(D);
871}
872
873static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
874 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
875 ValueProfData());
876}
877
878Error ValueProfData::checkIntegrity() {
879 if (NumValueKinds > IPVK_Last + 1)
880 return make_error<InstrProfError>(instrprof_error::malformed);
881 // Total size needs to be mulltiple of quadword size.
882 if (TotalSize % sizeof(uint64_t))
883 return make_error<InstrProfError>(instrprof_error::malformed);
884
885 ValueProfRecord *VR = getFirstValueProfRecord(this);
886 for (uint32_t K = 0; K < this->NumValueKinds; K++) {
887 if (VR->Kind > IPVK_Last)
888 return make_error<InstrProfError>(instrprof_error::malformed);
889 VR = getValueProfRecordNext(VR);
890 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
891 return make_error<InstrProfError>(instrprof_error::malformed);
892 }
893 return Error::success();
894}
895
896Expected<std::unique_ptr<ValueProfData>>
897ValueProfData::getValueProfData(const unsigned char *D,
898 const unsigned char *const BufferEnd,
899 support::endianness Endianness) {
900 using namespace support;
901
902 if (D + sizeof(ValueProfData) > BufferEnd)
903 return make_error<InstrProfError>(instrprof_error::truncated);
904
905 const unsigned char *Header = D;
906 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
907 if (D + TotalSize > BufferEnd)
908 return make_error<InstrProfError>(instrprof_error::too_large);
909
910 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
911 memcpy(VPD.get(), D, TotalSize);
912 // Byte swap.
913 VPD->swapBytesToHost(Endianness);
914
915 Error E = VPD->checkIntegrity();
916 if (E)
917 return std::move(E);
918
919 return std::move(VPD);
920}
921
922void ValueProfData::swapBytesToHost(support::endianness Endianness) {
923 using namespace support;
924
925 if (Endianness == getHostEndianness())
926 return;
927
928 sys::swapByteOrder<uint32_t>(TotalSize);
929 sys::swapByteOrder<uint32_t>(NumValueKinds);
930
931 ValueProfRecord *VR = getFirstValueProfRecord(this);
932 for (uint32_t K = 0; K < NumValueKinds; K++) {
933 VR->swapBytes(Endianness, getHostEndianness());
934 VR = getValueProfRecordNext(VR);
935 }
936}
937
938void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
939 using namespace support;
940
941 if (Endianness == getHostEndianness())
942 return;
943
944 ValueProfRecord *VR = getFirstValueProfRecord(this);
945 for (uint32_t K = 0; K < NumValueKinds; K++) {
946 ValueProfRecord *NVR = getValueProfRecordNext(VR);
947 VR->swapBytes(getHostEndianness(), Endianness);
948 VR = NVR;
949 }
950 sys::swapByteOrder<uint32_t>(TotalSize);
951 sys::swapByteOrder<uint32_t>(NumValueKinds);
952}
953
954void annotateValueSite(Module &M, Instruction &Inst,
955 const InstrProfRecord &InstrProfR,
956 InstrProfValueKind ValueKind, uint32_t SiteIdx,
957 uint32_t MaxMDCount) {
958 uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
959 if (!NV)
960 return;
961
962 uint64_t Sum = 0;
963 std::unique_ptr<InstrProfValueData[]> VD =
964 InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
965
966 ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
967 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
968}
969
970void annotateValueSite(Module &M, Instruction &Inst,
971 ArrayRef<InstrProfValueData> VDs,
972 uint64_t Sum, InstrProfValueKind ValueKind,
973 uint32_t MaxMDCount) {
974 LLVMContext &Ctx = M.getContext();
975 MDBuilder MDHelper(Ctx);
976 SmallVector<Metadata *, 3> Vals;
977 // Tag
978 Vals.push_back(MDHelper.createString("VP"));
979 // Value Kind
980 Vals.push_back(MDHelper.createConstant(
981 ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
982 // Total Count
983 Vals.push_back(
984 MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
985
986 // Value Profile Data
987 uint32_t MDCount = MaxMDCount;
988 for (auto &VD : VDs) {
989 Vals.push_back(MDHelper.createConstant(
990 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
991 Vals.push_back(MDHelper.createConstant(
992 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
993 if (--MDCount == 0)
994 break;
995 }
996 Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
997}
998
999bool getValueProfDataFromInst(const Instruction &Inst,
1000 InstrProfValueKind ValueKind,
1001 uint32_t MaxNumValueData,
1002 InstrProfValueData ValueData[],
1003 uint32_t &ActualNumValueData, uint64_t &TotalC,
1004 bool GetNoICPValue) {
1005 MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1006 if (!MD)
1007 return false;
1008
1009 unsigned NOps = MD->getNumOperands();
1010
1011 if (NOps < 5)
1012 return false;
1013
1014 // Operand 0 is a string tag "VP":
1015 MDString *Tag = cast<MDString>(MD->getOperand(0));
1016 if (!Tag)
1017 return false;
1018
1019 if (!Tag->getString().equals("VP"))
1020 return false;
1021
1022 // Now check kind:
1023 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1024 if (!KindInt)
1025 return false;
1026 if (KindInt->getZExtValue() != ValueKind)
1027 return false;
1028
1029 // Get total count
1030 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1031 if (!TotalCInt)
1032 return false;
1033 TotalC = TotalCInt->getZExtValue();
1034
1035 ActualNumValueData = 0;
1036
1037 for (unsigned I = 3; I < NOps; I += 2) {
1038 if (ActualNumValueData >= MaxNumValueData)
1039 break;
1040 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1041 ConstantInt *Count =
1042 mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1043 if (!Value || !Count)
1044 return false;
1045 uint64_t CntValue = Count->getZExtValue();
1046 if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1047 continue;
1048 ValueData[ActualNumValueData].Value = Value->getZExtValue();
1049 ValueData[ActualNumValueData].Count = CntValue;
1050 ActualNumValueData++;
1051 }
1052 return true;
1053}
1054
1055MDNode *getPGOFuncNameMetadata(const Function &F) {
1056 return F.getMetadata(getPGOFuncNameMetadataName());
1057}
1058
1059void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
1060 // Only for internal linkage functions.
1061 if (PGOFuncName == F.getName())
1062 return;
1063 // Don't create duplicated meta-data.
1064 if (getPGOFuncNameMetadata(F))
1065 return;
1066 LLVMContext &C = F.getContext();
1067 MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1068 F.setMetadata(getPGOFuncNameMetadataName(), N);
1069}
1070
1071bool needsComdatForCounter(const Function &F, const Module &M) {
1072 if (F.hasComdat())
1073 return true;
1074
1075 if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1076 return false;
1077
1078 // See createPGOFuncNameVar for more details. To avoid link errors, profile
1079 // counters for function with available_externally linkage needs to be changed
1080 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1081 // created. Without using comdat, duplicate entries won't be removed by the
1082 // linker leading to increased data segement size and raw profile size. Even
1083 // worse, since the referenced counter from profile per-function data object
1084 // will be resolved to the common strong definition, the profile counts for
1085 // available_externally functions will end up being duplicated in raw profile
1086 // data. This can result in distorted profile as the counts of those dups
1087 // will be accumulated by the profile merger.
1088 GlobalValue::LinkageTypes Linkage = F.getLinkage();
1089 if (Linkage != GlobalValue::ExternalWeakLinkage &&
1090 Linkage != GlobalValue::AvailableExternallyLinkage)
1091 return false;
1092
1093 return true;
1094}
1095
1096// Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1097bool isIRPGOFlagSet(const Module *M) {
1098 auto IRInstrVar =
1099 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version");
1100 if (!IRInstrVar || IRInstrVar->isDeclaration() ||
1101 IRInstrVar->hasLocalLinkage())
1102 return false;
1103
1104 // Check if the flag is set.
1105 if (!IRInstrVar->hasInitializer())
1106 return false;
1107
1108 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1109 if (!InitVal)
1110 return false;
1111 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF(0x1ULL << 56)) != 0;
1112}
1113
1114// Check if we can safely rename this Comdat function.
1115bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1116 if (F.getName().empty())
1117 return false;
1118 if (!needsComdatForCounter(F, *(F.getParent())))
1119 return false;
1120 // Unsafe to rename the address-taken function (which can be used in
1121 // function comparison).
1122 if (CheckAddressTaken && F.hasAddressTaken())
1123 return false;
1124 // Only safe to do if this function may be discarded if it is not used
1125 // in the compilation unit.
1126 if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1127 return false;
1128
1129 // For AvailableExternallyLinkage functions.
1130 if (!F.hasComdat()) {
1131 assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage)((F.getLinkage() == GlobalValue::AvailableExternallyLinkage) ?
static_cast<void> (0) : __assert_fail ("F.getLinkage() == GlobalValue::AvailableExternallyLinkage"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/ProfileData/InstrProf.cpp"
, 1131, __PRETTY_FUNCTION__))
;
1132 return true;
1133 }
1134 return true;
1135}
1136
1137// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1138// aware this is an ir_level profile so it can set the version flag.
1139void createIRLevelProfileFlagVar(Module &M, bool IsCS,
1140 bool InstrEntryBBEnabled) {
1141 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version");
1142 Type *IntTy64 = Type::getInt64Ty(M.getContext());
1143 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION5 | VARIANT_MASK_IR_PROF(0x1ULL << 56));
1144 if (IsCS)
1145 ProfileVersion |= VARIANT_MASK_CSIR_PROF(0x1ULL << 57);
1146 if (InstrEntryBBEnabled)
1147 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY(0x1ULL << 58);
1148 auto IRLevelVersionVariable = new GlobalVariable(
1149 M, IntTy64, true, GlobalValue::WeakAnyLinkage,
1150 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
1151 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
1152 Triple TT(M.getTargetTriple());
1153 if (TT.supportsCOMDAT()) {
1154 IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage);
1155 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
1156 }
1157}
1158
1159// Create the variable for the profile file name.
1160void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1161 if (InstrProfileOutput.empty())
1162 return;
1163 Constant *ProfileNameConst =
1164 ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1165 GlobalVariable *ProfileNameVar = new GlobalVariable(
1166 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1167 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename");
1168 Triple TT(M.getTargetTriple());
1169 if (TT.supportsCOMDAT()) {
1170 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
1171 ProfileNameVar->setComdat(M.getOrInsertComdat(
1172 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename")));
1173 }
1174}
1175
1176Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1177 const std::string &TestFilename,
1178 bool IsCS) {
1179 auto getProfileSum = [IsCS](const std::string &Filename,
1180 CountSumOrPercent &Sum) -> Error {
1181 auto ReaderOrErr = InstrProfReader::create(Filename);
1182 if (Error E = ReaderOrErr.takeError()) {
1183 return E;
1184 }
1185 auto Reader = std::move(ReaderOrErr.get());
1186 Reader->accumulateCounts(Sum, IsCS);
1187 return Error::success();
1188 };
1189 auto Ret = getProfileSum(BaseFilename, Base);
1190 if (Ret)
1191 return Ret;
1192 Ret = getProfileSum(TestFilename, Test);
1193 if (Ret)
1194 return Ret;
1195 this->BaseFilename = &BaseFilename;
1196 this->TestFilename = &TestFilename;
1197 Valid = true;
1198 return Error::success();
1199}
1200
1201void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) {
1202 Mismatch.NumEntries += 1;
1203 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1204 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1205 if (Test.ValueCounts[I] >= 1.0f)
1206 Mismatch.ValueCounts[I] +=
1207 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1208 }
1209}
1210
1211void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) {
1212 Unique.NumEntries += 1;
1213 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1214 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1215 if (Test.ValueCounts[I] >= 1.0f)
1216 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1217 }
1218}
1219
1220void OverlapStats::dump(raw_fd_ostream &OS) const {
1221 if (!Valid)
1222 return;
1223
1224 const char *EntryName =
1225 (Level == ProgramLevel ? "functions" : "edge counters");
1226 if (Level == ProgramLevel) {
1227 OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1228 << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1229 } else {
1230 OS << "Function level:\n"
1231 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1232 }
1233
1234 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1235 if (Mismatch.NumEntries)
1236 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1237 << "\n";
1238 if (Unique.NumEntries)
1239 OS << " # of " << EntryName
1240 << " only in test_profile: " << Unique.NumEntries << "\n";
1241
1242 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1243 << "\n";
1244 if (Mismatch.NumEntries)
1245 OS << " Mismatched count percentage (Edge): "
1246 << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1247 if (Unique.NumEntries)
1248 OS << " Percentage of Edge profile only in test_profile: "
1249 << format("%.3f%%", Unique.CountSum * 100) << "\n";
1250 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1251 << "\n"
1252 << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1253 << "\n";
1254
1255 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1256 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1257 continue;
1258 char ProfileKindName[20];
1259 switch (I) {
1260 case IPVK_IndirectCallTarget:
1261 strncpy(ProfileKindName, "IndirectCall", 19);
1262 break;
1263 case IPVK_MemOPSize:
1264 strncpy(ProfileKindName, "MemOP", 19);
1265 break;
1266 default:
1267 snprintf(ProfileKindName, 19, "VP[%d]", I);
1268 break;
1269 }
1270 OS << " " << ProfileKindName
1271 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1272 << "\n";
1273 if (Mismatch.NumEntries)
1274 OS << " Mismatched count percentage (" << ProfileKindName
1275 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1276 if (Unique.NumEntries)
1277 OS << " Percentage of " << ProfileKindName
1278 << " profile only in test_profile: "
1279 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1280 OS << " " << ProfileKindName
1281 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1282 << "\n"
1283 << " " << ProfileKindName
1284 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1285 << "\n";
1286 }
1287}
1288
1289} // end namespace llvm

/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/Support/LEB128.h

1//===- llvm/Support/LEB128.h - [SU]LEB128 utility functions -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares some utility functions for encoding SLEB128 and
10// ULEB128 values.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_LEB128_H
15#define LLVM_SUPPORT_LEB128_H
16
17#include "llvm/Support/raw_ostream.h"
18
19namespace llvm {
20
21/// Utility function to encode a SLEB128 value to an output stream. Returns
22/// the length in bytes of the encoded value.
23inline unsigned encodeSLEB128(int64_t Value, raw_ostream &OS,
24 unsigned PadTo = 0) {
25 bool More;
26 unsigned Count = 0;
27 do {
28 uint8_t Byte = Value & 0x7f;
29 // NOTE: this assumes that this signed shift is an arithmetic right shift.
30 Value >>= 7;
31 More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) ||
32 ((Value == -1) && ((Byte & 0x40) != 0))));
33 Count++;
34 if (More || Count < PadTo)
35 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
36 OS << char(Byte);
37 } while (More);
38
39 // Pad with 0x80 and emit a terminating byte at the end.
40 if (Count < PadTo) {
41 uint8_t PadValue = Value < 0 ? 0x7f : 0x00;
42 for (; Count < PadTo - 1; ++Count)
43 OS << char(PadValue | 0x80);
44 OS << char(PadValue);
45 Count++;
46 }
47 return Count;
48}
49
50/// Utility function to encode a SLEB128 value to a buffer. Returns
51/// the length in bytes of the encoded value.
52inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned PadTo = 0) {
53 uint8_t *orig_p = p;
54 unsigned Count = 0;
55 bool More;
56 do {
57 uint8_t Byte = Value & 0x7f;
58 // NOTE: this assumes that this signed shift is an arithmetic right shift.
59 Value >>= 7;
60 More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) ||
61 ((Value == -1) && ((Byte & 0x40) != 0))));
62 Count++;
63 if (More || Count < PadTo)
64 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
65 *p++ = Byte;
66 } while (More);
67
68 // Pad with 0x80 and emit a terminating byte at the end.
69 if (Count < PadTo) {
70 uint8_t PadValue = Value < 0 ? 0x7f : 0x00;
71 for (; Count < PadTo - 1; ++Count)
72 *p++ = (PadValue | 0x80);
73 *p++ = PadValue;
74 }
75 return (unsigned)(p - orig_p);
76}
77
78/// Utility function to encode a ULEB128 value to an output stream. Returns
79/// the length in bytes of the encoded value.
80inline unsigned encodeULEB128(uint64_t Value, raw_ostream &OS,
81 unsigned PadTo = 0) {
82 unsigned Count = 0;
83 do {
84 uint8_t Byte = Value & 0x7f;
85 Value >>= 7;
86 Count++;
87 if (Value != 0 || Count < PadTo)
88 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
89 OS << char(Byte);
90 } while (Value != 0);
91
92 // Pad with 0x80 and emit a null byte at the end.
93 if (Count < PadTo) {
94 for (; Count < PadTo - 1; ++Count)
95 OS << '\x80';
96 OS << '\x00';
97 Count++;
98 }
99 return Count;
100}
101
102/// Utility function to encode a ULEB128 value to a buffer. Returns
103/// the length in bytes of the encoded value.
104inline unsigned encodeULEB128(uint64_t Value, uint8_t *p,
105 unsigned PadTo = 0) {
106 uint8_t *orig_p = p;
107 unsigned Count = 0;
108 do {
109 uint8_t Byte = Value & 0x7f;
110 Value >>= 7;
111 Count++;
112 if (Value != 0 || Count < PadTo)
113 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
114 *p++ = Byte;
115 } while (Value != 0);
116
117 // Pad with 0x80 and emit a null byte at the end.
118 if (Count < PadTo) {
119 for (; Count < PadTo - 1; ++Count)
120 *p++ = '\x80';
121 *p++ = '\x00';
122 }
123
124 return (unsigned)(p - orig_p);
125}
126
127/// Utility function to decode a ULEB128 value.
128inline uint64_t decodeULEB128(const uint8_t *p, unsigned *n = nullptr,
129 const uint8_t *end = nullptr,
130 const char **error = nullptr) {
131 const uint8_t *orig_p = p;
132 uint64_t Value = 0;
133 unsigned Shift = 0;
134 if (error
3.1
'error' is null
3.1
'error' is null
)
4
Taking false branch
135 *error = nullptr;
136 do {
137 if (p == end) {
5
Assuming 'p' is equal to 'end'
6
Taking true branch
138 if (error
6.1
'error' is null
6.1
'error' is null
)
7
Taking false branch
139 *error = "malformed uleb128, extends past end";
140 if (n
7.1
'n' is non-null
7.1
'n' is non-null
)
8
Taking true branch
141 *n = (unsigned)(p - orig_p);
142 return 0;
9
Returning zero
143 }
144 uint64_t Slice = *p & 0x7f;
145 if ((Shift >= 64 && Slice != 0) || Slice << Shift >> Shift != Slice) {
146 if (error)
147 *error = "uleb128 too big for uint64";
148 if (n)
149 *n = (unsigned)(p - orig_p);
150 return 0;
151 }
152 Value += Slice << Shift;
153 Shift += 7;
154 } while (*p++ >= 128);
155 if (n)
156 *n = (unsigned)(p - orig_p);
157 return Value;
158}
159
160/// Utility function to decode a SLEB128 value.
161inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr,
162 const uint8_t *end = nullptr,
163 const char **error = nullptr) {
164 const uint8_t *orig_p = p;
165 int64_t Value = 0;
166 unsigned Shift = 0;
167 uint8_t Byte;
168 if (error)
169 *error = nullptr;
170 do {
171 if (p == end) {
172 if (error)
173 *error = "malformed sleb128, extends past end";
174 if (n)
175 *n = (unsigned)(p - orig_p);
176 return 0;
177 }
178 Byte = *p;
179 uint64_t Slice = Byte & 0x7f;
180 if ((Shift >= 64 && Slice != (Value < 0 ? 0x7f : 0x00)) ||
181 (Shift == 63 && Slice != 0 && Slice != 0x7f)) {
182 if (error)
183 *error = "sleb128 too big for int64";
184 if (n)
185 *n = (unsigned)(p - orig_p);
186 return 0;
187 }
188 Value |= Slice << Shift;
189 Shift += 7;
190 ++p;
191 } while (Byte >= 128);
192 // Sign extend negative numbers if needed.
193 if (Shift < 64 && (Byte & 0x40))
194 Value |= (-1ULL) << Shift;
195 if (n)
196 *n = (unsigned)(p - orig_p);
197 return Value;
198}
199
200/// Utility function to get the size of the ULEB128-encoded value.
201extern unsigned getULEB128Size(uint64_t Value);
202
203/// Utility function to get the size of the SLEB128-encoded value.
204extern unsigned getSLEB128Size(int64_t Value);
205
206} // namespace llvm
207
208#endif // LLVM_SUPPORT_LEB128_H