Bug Summary

File:llvm/lib/ProfileData/InstrProf.cpp
Warning:line 479, column 24
Dereference of null pointer (loaded from variable 'P')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name InstrProf.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/ProfileData -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/ProfileData -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-09-26-161721-17566-1 -x c++ /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp

/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp

1//===- InstrProf.cpp - Instrumented profiling format support --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for clang's instrumentation based PGO and
10// coverage.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/ProfileData/InstrProf.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallString.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Triple.h"
21#include "llvm/IR/Constant.h"
22#include "llvm/IR/Constants.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/GlobalValue.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/LLVMContext.h"
28#include "llvm/IR/MDBuilder.h"
29#include "llvm/IR/Metadata.h"
30#include "llvm/IR/Module.h"
31#include "llvm/IR/Type.h"
32#include "llvm/ProfileData/InstrProfReader.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Compiler.h"
36#include "llvm/Support/Compression.h"
37#include "llvm/Support/Endian.h"
38#include "llvm/Support/Error.h"
39#include "llvm/Support/ErrorHandling.h"
40#include "llvm/Support/LEB128.h"
41#include "llvm/Support/ManagedStatic.h"
42#include "llvm/Support/MathExtras.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SwapByteOrder.h"
45#include <algorithm>
46#include <cassert>
47#include <cstddef>
48#include <cstdint>
49#include <cstring>
50#include <memory>
51#include <string>
52#include <system_error>
53#include <utility>
54#include <vector>
55
56using namespace llvm;
57
58static cl::opt<bool> StaticFuncFullModulePrefix(
59 "static-func-full-module-prefix", cl::init(true), cl::Hidden,
60 cl::desc("Use full module build paths in the profile counter names for "
61 "static functions."));
62
63// This option is tailored to users that have different top-level directory in
64// profile-gen and profile-use compilation. Users need to specific the number
65// of levels to strip. A value larger than the number of directories in the
66// source file will strip all the directory names and only leave the basename.
67//
68// Note current ThinLTO module importing for the indirect-calls assumes
69// the source directory name not being stripped. A non-zero option value here
70// can potentially prevent some inter-module indirect-call-promotions.
71static cl::opt<unsigned> StaticFuncStripDirNamePrefix(
72 "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
73 cl::desc("Strip specified level of directory name from source path in "
74 "the profile counter name for static functions."));
75
76static std::string getInstrProfErrString(instrprof_error Err) {
77 switch (Err) {
78 case instrprof_error::success:
79 return "Success";
80 case instrprof_error::eof:
81 return "End of File";
82 case instrprof_error::unrecognized_format:
83 return "Unrecognized instrumentation profile encoding format";
84 case instrprof_error::bad_magic:
85 return "Invalid instrumentation profile data (bad magic)";
86 case instrprof_error::bad_header:
87 return "Invalid instrumentation profile data (file header is corrupt)";
88 case instrprof_error::unsupported_version:
89 return "Unsupported instrumentation profile format version";
90 case instrprof_error::unsupported_hash_type:
91 return "Unsupported instrumentation profile hash type";
92 case instrprof_error::too_large:
93 return "Too much profile data";
94 case instrprof_error::truncated:
95 return "Truncated profile data";
96 case instrprof_error::malformed:
97 return "Malformed instrumentation profile data";
98 case instrprof_error::unknown_function:
99 return "No profile data available for function";
100 case instrprof_error::hash_mismatch:
101 return "Function control flow change detected (hash mismatch)";
102 case instrprof_error::count_mismatch:
103 return "Function basic block count change detected (counter mismatch)";
104 case instrprof_error::counter_overflow:
105 return "Counter overflow";
106 case instrprof_error::value_site_count_mismatch:
107 return "Function value site count change detected (counter mismatch)";
108 case instrprof_error::compress_failed:
109 return "Failed to compress data (zlib)";
110 case instrprof_error::uncompress_failed:
111 return "Failed to uncompress data (zlib)";
112 case instrprof_error::empty_raw_profile:
113 return "Empty raw profile file";
114 case instrprof_error::zlib_unavailable:
115 return "Profile uses zlib compression but the profile reader was built without zlib support";
116 }
117 llvm_unreachable("A value of instrprof_error has no message.")::llvm::llvm_unreachable_internal("A value of instrprof_error has no message."
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 117)
;
118}
119
120namespace {
121
122// FIXME: This class is only here to support the transition to llvm::Error. It
123// will be removed once this transition is complete. Clients should prefer to
124// deal with the Error value directly, rather than converting to error_code.
125class InstrProfErrorCategoryType : public std::error_category {
126 const char *name() const noexcept override { return "llvm.instrprof"; }
127
128 std::string message(int IE) const override {
129 return getInstrProfErrString(static_cast<instrprof_error>(IE));
130 }
131};
132
133} // end anonymous namespace
134
135static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
136
137const std::error_category &llvm::instrprof_category() {
138 return *ErrorCategory;
139}
140
141namespace {
142
143const char *InstrProfSectNameCommon[] = {
144#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
145 SectNameCommon,
146#include "llvm/ProfileData/InstrProfData.inc"
147};
148
149const char *InstrProfSectNameCoff[] = {
150#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
151 SectNameCoff,
152#include "llvm/ProfileData/InstrProfData.inc"
153};
154
155const char *InstrProfSectNamePrefix[] = {
156#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
157 Prefix,
158#include "llvm/ProfileData/InstrProfData.inc"
159};
160
161} // namespace
162
163namespace llvm {
164
165cl::opt<bool> DoInstrProfNameCompression(
166 "enable-name-compression",
167 cl::desc("Enable name/filename string compression"), cl::init(true));
168
169std::string getInstrProfSectionName(InstrProfSectKind IPSK,
170 Triple::ObjectFormatType OF,
171 bool AddSegmentInfo) {
172 std::string SectName;
173
174 if (OF == Triple::MachO && AddSegmentInfo)
175 SectName = InstrProfSectNamePrefix[IPSK];
176
177 if (OF == Triple::COFF)
178 SectName += InstrProfSectNameCoff[IPSK];
179 else
180 SectName += InstrProfSectNameCommon[IPSK];
181
182 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
183 SectName += ",regular,live_support";
184
185 return SectName;
186}
187
188void SoftInstrProfErrors::addError(instrprof_error IE) {
189 if (IE == instrprof_error::success)
190 return;
191
192 if (FirstError == instrprof_error::success)
193 FirstError = IE;
194
195 switch (IE) {
196 case instrprof_error::hash_mismatch:
197 ++NumHashMismatches;
198 break;
199 case instrprof_error::count_mismatch:
200 ++NumCountMismatches;
201 break;
202 case instrprof_error::counter_overflow:
203 ++NumCounterOverflows;
204 break;
205 case instrprof_error::value_site_count_mismatch:
206 ++NumValueSiteCountMismatches;
207 break;
208 default:
209 llvm_unreachable("Not a soft error")::llvm::llvm_unreachable_internal("Not a soft error", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 209)
;
210 }
211}
212
213std::string InstrProfError::message() const {
214 return getInstrProfErrString(Err);
215}
216
217char InstrProfError::ID = 0;
218
219std::string getPGOFuncName(StringRef RawFuncName,
220 GlobalValue::LinkageTypes Linkage,
221 StringRef FileName,
222 uint64_t Version LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__))) {
223 return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
224}
225
226// Strip NumPrefix level of directory name from PathNameStr. If the number of
227// directory separators is less than NumPrefix, strip all the directories and
228// leave base file name only.
229static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
230 uint32_t Count = NumPrefix;
231 uint32_t Pos = 0, LastPos = 0;
232 for (auto & CI : PathNameStr) {
233 ++Pos;
234 if (llvm::sys::path::is_separator(CI)) {
235 LastPos = Pos;
236 --Count;
237 }
238 if (Count == 0)
239 break;
240 }
241 return PathNameStr.substr(LastPos);
242}
243
244// Return the PGOFuncName. This function has some special handling when called
245// in LTO optimization. The following only applies when calling in LTO passes
246// (when \c InLTO is true): LTO's internalization privatizes many global linkage
247// symbols. This happens after value profile annotation, but those internal
248// linkage functions should not have a source prefix.
249// Additionally, for ThinLTO mode, exported internal functions are promoted
250// and renamed. We need to ensure that the original internal PGO name is
251// used when computing the GUID that is compared against the profiled GUIDs.
252// To differentiate compiler generated internal symbols from original ones,
253// PGOFuncName meta data are created and attached to the original internal
254// symbols in the value profile annotation step
255// (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
256// data, its original linkage must be non-internal.
257std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
258 if (!InLTO) {
259 StringRef FileName(F.getParent()->getSourceFileName());
260 uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
261 if (StripLevel < StaticFuncStripDirNamePrefix)
262 StripLevel = StaticFuncStripDirNamePrefix;
263 if (StripLevel)
264 FileName = stripDirPrefix(FileName, StripLevel);
265 return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
266 }
267
268 // In LTO mode (when InLTO is true), first check if there is a meta data.
269 if (MDNode *MD = getPGOFuncNameMetadata(F)) {
270 StringRef S = cast<MDString>(MD->getOperand(0))->getString();
271 return S.str();
272 }
273
274 // If there is no meta data, the function must be a global before the value
275 // profile annotation pass. Its current linkage may be internal if it is
276 // internalized in LTO mode.
277 return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
278}
279
280StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
281 if (FileName.empty())
282 return PGOFuncName;
283 // Drop the file name including ':'. See also getPGOFuncName.
284 if (PGOFuncName.startswith(FileName))
285 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
286 return PGOFuncName;
287}
288
289// \p FuncName is the string used as profile lookup key for the function. A
290// symbol is created to hold the name. Return the legalized symbol name.
291std::string getPGOFuncNameVarName(StringRef FuncName,
292 GlobalValue::LinkageTypes Linkage) {
293 std::string VarName = std::string(getInstrProfNameVarPrefix());
294 VarName += FuncName;
295
296 if (!GlobalValue::isLocalLinkage(Linkage))
297 return VarName;
298
299 // Now fix up illegal chars in local VarName that may upset the assembler.
300 const char *InvalidChars = "-:<>/\"'";
301 size_t found = VarName.find_first_of(InvalidChars);
302 while (found != std::string::npos) {
303 VarName[found] = '_';
304 found = VarName.find_first_of(InvalidChars, found + 1);
305 }
306 return VarName;
307}
308
309GlobalVariable *createPGOFuncNameVar(Module &M,
310 GlobalValue::LinkageTypes Linkage,
311 StringRef PGOFuncName) {
312 // We generally want to match the function's linkage, but available_externally
313 // and extern_weak both have the wrong semantics, and anything that doesn't
314 // need to link across compilation units doesn't need to be visible at all.
315 if (Linkage == GlobalValue::ExternalWeakLinkage)
316 Linkage = GlobalValue::LinkOnceAnyLinkage;
317 else if (Linkage == GlobalValue::AvailableExternallyLinkage)
318 Linkage = GlobalValue::LinkOnceODRLinkage;
319 else if (Linkage == GlobalValue::InternalLinkage ||
320 Linkage == GlobalValue::ExternalLinkage)
321 Linkage = GlobalValue::PrivateLinkage;
322
323 auto *Value =
324 ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
325 auto FuncNameVar =
326 new GlobalVariable(M, Value->getType(), true, Linkage, Value,
327 getPGOFuncNameVarName(PGOFuncName, Linkage));
328
329 // Hide the symbol so that we correctly get a copy for each executable.
330 if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
331 FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
332
333 return FuncNameVar;
334}
335
336GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
337 return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
338}
339
340Error InstrProfSymtab::create(Module &M, bool InLTO) {
341 for (Function &F : M) {
342 // Function may not have a name: like using asm("") to overwrite the name.
343 // Ignore in this case.
344 if (!F.hasName())
345 continue;
346 const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
347 if (Error E = addFuncName(PGOFuncName))
348 return E;
349 MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
350 // In ThinLTO, local function may have been promoted to global and have
351 // suffix added to the function name. We need to add the stripped function
352 // name to the symbol table so that we can find a match from profile.
353 if (InLTO) {
354 auto pos = PGOFuncName.find('.');
355 if (pos != std::string::npos) {
356 const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
357 if (Error E = addFuncName(OtherFuncName))
358 return E;
359 MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
360 }
361 }
362 }
363 Sorted = false;
364 finalizeSymtab();
365 return Error::success();
366}
367
368uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
369 finalizeSymtab();
370 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
371 return A.first < Address;
372 });
373 // Raw function pointer collected by value profiler may be from
374 // external functions that are not instrumented. They won't have
375 // mapping data to be used by the deserializer. Force the value to
376 // be 0 in this case.
377 if (It != AddrToMD5Map.end() && It->first == Address)
378 return (uint64_t)It->second;
379 return 0;
380}
381
382Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
383 bool doCompression, std::string &Result) {
384 assert(!NameStrs.empty() && "No name data to emit")((!NameStrs.empty() && "No name data to emit") ? static_cast
<void> (0) : __assert_fail ("!NameStrs.empty() && \"No name data to emit\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 384, __PRETTY_FUNCTION__))
;
385
386 uint8_t Header[16], *P = Header;
387 std::string UncompressedNameStrings =
388 join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
389
390 assert(StringRef(UncompressedNameStrings)((StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator
()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)"
) ? static_cast<void> (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 392, __PRETTY_FUNCTION__))
391 .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&((StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator
()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)"
) ? static_cast<void> (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 392, __PRETTY_FUNCTION__))
392 "PGO name is invalid (contains separator token)")((StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator
()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)"
) ? static_cast<void> (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 392, __PRETTY_FUNCTION__))
;
393
394 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
395 P += EncLen;
396
397 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
398 EncLen = encodeULEB128(CompressedLen, P);
399 P += EncLen;
400 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
401 unsigned HeaderLen = P - &Header[0];
402 Result.append(HeaderStr, HeaderLen);
403 Result += InputStr;
404 return Error::success();
405 };
406
407 if (!doCompression) {
408 return WriteStringToResult(0, UncompressedNameStrings);
409 }
410
411 SmallString<128> CompressedNameStrings;
412 Error E = zlib::compress(StringRef(UncompressedNameStrings),
413 CompressedNameStrings, zlib::BestSizeCompression);
414 if (E) {
415 consumeError(std::move(E));
416 return make_error<InstrProfError>(instrprof_error::compress_failed);
417 }
418
419 return WriteStringToResult(CompressedNameStrings.size(),
420 CompressedNameStrings);
421}
422
423StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
424 auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
425 StringRef NameStr =
426 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
427 return NameStr;
428}
429
430Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
431 std::string &Result, bool doCompression) {
432 std::vector<std::string> NameStrs;
433 for (auto *NameVar : NameVars) {
434 NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
435 }
436 return collectPGOFuncNameStrings(
437 NameStrs, zlib::isAvailable() && doCompression, Result);
438}
439
440Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
441 const uint8_t *P = NameStrings.bytes_begin();
442 const uint8_t *EndP = NameStrings.bytes_end();
443 while (P < EndP) {
1
Assuming 'P' is < 'EndP'
2
Loop condition is true. Entering loop body
444 uint32_t N;
445 uint64_t UncompressedSize = decodeULEB128(P, &N);
3
Calling 'decodeULEB128'
10
Returning from 'decodeULEB128'
11
'UncompressedSize' initialized to 0
446 P += N;
447 uint64_t CompressedSize = decodeULEB128(P, &N);
448 P += N;
449 bool isCompressed = (CompressedSize != 0);
450 SmallString<128> UncompressedNameStrings;
451 StringRef NameStrings;
452 if (isCompressed
11.1
'isCompressed' is false
11.1
'isCompressed' is false
) {
12
Taking false branch
453 if (!llvm::zlib::isAvailable())
454 return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
455
456 StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
457 CompressedSize);
458 if (Error E =
459 zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
460 UncompressedSize)) {
461 consumeError(std::move(E));
462 return make_error<InstrProfError>(instrprof_error::uncompress_failed);
463 }
464 P += CompressedSize;
465 NameStrings = StringRef(UncompressedNameStrings.data(),
466 UncompressedNameStrings.size());
467 } else {
468 NameStrings =
469 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
470 P += UncompressedSize;
13
Null pointer value stored to 'P'
471 }
472 // Now parse the name strings.
473 SmallVector<StringRef, 0> Names;
474 NameStrings.split(Names, getInstrProfNameSeparator());
475 for (StringRef &Name : Names)
14
Assuming '__begin2' is equal to '__end2'
476 if (Error E = Symtab.addFuncName(Name))
477 return E;
478
479 while (P
14.1
'P' is < 'EndP'
14.1
'P' is < 'EndP'
< EndP && *P == 0)
15
Dereference of null pointer (loaded from variable 'P')
480 P++;
481 }
482 return Error::success();
483}
484
485void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
486 uint64_t FuncSum = 0;
487 Sum.NumEntries += Counts.size();
488 for (size_t F = 0, E = Counts.size(); F < E; ++F)
489 FuncSum += Counts[F];
490 Sum.CountSum += FuncSum;
491
492 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
493 uint64_t KindSum = 0;
494 uint32_t NumValueSites = getNumValueSites(VK);
495 for (size_t I = 0; I < NumValueSites; ++I) {
496 uint32_t NV = getNumValueDataForSite(VK, I);
497 std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I);
498 for (uint32_t V = 0; V < NV; V++)
499 KindSum += VD[V].Count;
500 }
501 Sum.ValueCounts[VK] += KindSum;
502 }
503}
504
505void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input,
506 uint32_t ValueKind,
507 OverlapStats &Overlap,
508 OverlapStats &FuncLevelOverlap) {
509 this->sortByTargetValues();
510 Input.sortByTargetValues();
511 double Score = 0.0f, FuncLevelScore = 0.0f;
512 auto I = ValueData.begin();
513 auto IE = ValueData.end();
514 auto J = Input.ValueData.begin();
515 auto JE = Input.ValueData.end();
516 while (I != IE && J != JE) {
517 if (I->Value == J->Value) {
518 Score += OverlapStats::score(I->Count, J->Count,
519 Overlap.Base.ValueCounts[ValueKind],
520 Overlap.Test.ValueCounts[ValueKind]);
521 FuncLevelScore += OverlapStats::score(
522 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
523 FuncLevelOverlap.Test.ValueCounts[ValueKind]);
524 ++I;
525 } else if (I->Value < J->Value) {
526 ++I;
527 continue;
528 }
529 ++J;
530 }
531 Overlap.Overlap.ValueCounts[ValueKind] += Score;
532 FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
533}
534
535// Return false on mismatch.
536void InstrProfRecord::overlapValueProfData(uint32_t ValueKind,
537 InstrProfRecord &Other,
538 OverlapStats &Overlap,
539 OverlapStats &FuncLevelOverlap) {
540 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
541 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind))((ThisNumValueSites == Other.getNumValueSites(ValueKind)) ? static_cast
<void> (0) : __assert_fail ("ThisNumValueSites == Other.getNumValueSites(ValueKind)"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 541, __PRETTY_FUNCTION__))
;
542 if (!ThisNumValueSites)
543 return;
544
545 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
546 getOrCreateValueSitesForKind(ValueKind);
547 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
548 Other.getValueSitesForKind(ValueKind);
549 for (uint32_t I = 0; I < ThisNumValueSites; I++)
550 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
551 FuncLevelOverlap);
552}
553
554void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap,
555 OverlapStats &FuncLevelOverlap,
556 uint64_t ValueCutoff) {
557 // FuncLevel CountSum for other should already computed and nonzero.
558 assert(FuncLevelOverlap.Test.CountSum >= 1.0f)((FuncLevelOverlap.Test.CountSum >= 1.0f) ? static_cast<
void> (0) : __assert_fail ("FuncLevelOverlap.Test.CountSum >= 1.0f"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 558, __PRETTY_FUNCTION__))
;
559 accumulateCounts(FuncLevelOverlap.Base);
560 bool Mismatch = (Counts.size() != Other.Counts.size());
561
562 // Check if the value profiles mismatch.
563 if (!Mismatch) {
564 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
565 uint32_t ThisNumValueSites = getNumValueSites(Kind);
566 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
567 if (ThisNumValueSites != OtherNumValueSites) {
568 Mismatch = true;
569 break;
570 }
571 }
572 }
573 if (Mismatch) {
574 Overlap.addOneMismatch(FuncLevelOverlap.Test);
575 return;
576 }
577
578 // Compute overlap for value counts.
579 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
580 overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
581
582 double Score = 0.0;
583 uint64_t MaxCount = 0;
584 // Compute overlap for edge counts.
585 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
586 Score += OverlapStats::score(Counts[I], Other.Counts[I],
587 Overlap.Base.CountSum, Overlap.Test.CountSum);
588 MaxCount = std::max(Other.Counts[I], MaxCount);
589 }
590 Overlap.Overlap.CountSum += Score;
591 Overlap.Overlap.NumEntries += 1;
592
593 if (MaxCount >= ValueCutoff) {
594 double FuncScore = 0.0;
595 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
596 FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
597 FuncLevelOverlap.Base.CountSum,
598 FuncLevelOverlap.Test.CountSum);
599 FuncLevelOverlap.Overlap.CountSum = FuncScore;
600 FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
601 FuncLevelOverlap.Valid = true;
602 }
603}
604
605void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
606 uint64_t Weight,
607 function_ref<void(instrprof_error)> Warn) {
608 this->sortByTargetValues();
609 Input.sortByTargetValues();
610 auto I = ValueData.begin();
611 auto IE = ValueData.end();
612 for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
613 ++J) {
614 while (I != IE && I->Value < J->Value)
615 ++I;
616 if (I != IE && I->Value == J->Value) {
617 bool Overflowed;
618 I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed);
619 if (Overflowed)
620 Warn(instrprof_error::counter_overflow);
621 ++I;
622 continue;
623 }
624 ValueData.insert(I, *J);
625 }
626}
627
628void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D,
629 function_ref<void(instrprof_error)> Warn) {
630 for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
631 bool Overflowed;
632 I->Count = SaturatingMultiply(I->Count, N, &Overflowed) / D;
633 if (Overflowed)
634 Warn(instrprof_error::counter_overflow);
635 }
636}
637
638// Merge Value Profile data from Src record to this record for ValueKind.
639// Scale merged value counts by \p Weight.
640void InstrProfRecord::mergeValueProfData(
641 uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
642 function_ref<void(instrprof_error)> Warn) {
643 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
644 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
645 if (ThisNumValueSites != OtherNumValueSites) {
646 Warn(instrprof_error::value_site_count_mismatch);
647 return;
648 }
649 if (!ThisNumValueSites)
650 return;
651 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
652 getOrCreateValueSitesForKind(ValueKind);
653 MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
654 Src.getValueSitesForKind(ValueKind);
655 for (uint32_t I = 0; I < ThisNumValueSites; I++)
656 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
657}
658
659void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
660 function_ref<void(instrprof_error)> Warn) {
661 // If the number of counters doesn't match we either have bad data
662 // or a hash collision.
663 if (Counts.size() != Other.Counts.size()) {
664 Warn(instrprof_error::count_mismatch);
665 return;
666 }
667
668 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
669 bool Overflowed;
670 Counts[I] =
671 SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
672 if (Overflowed)
673 Warn(instrprof_error::counter_overflow);
674 }
675
676 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
677 mergeValueProfData(Kind, Other, Weight, Warn);
678}
679
680void InstrProfRecord::scaleValueProfData(
681 uint32_t ValueKind, uint64_t N, uint64_t D,
682 function_ref<void(instrprof_error)> Warn) {
683 for (auto &R : getValueSitesForKind(ValueKind))
684 R.scale(N, D, Warn);
685}
686
687void InstrProfRecord::scale(uint64_t N, uint64_t D,
688 function_ref<void(instrprof_error)> Warn) {
689 assert(D != 0 && "D cannot be 0")((D != 0 && "D cannot be 0") ? static_cast<void>
(0) : __assert_fail ("D != 0 && \"D cannot be 0\"", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 689, __PRETTY_FUNCTION__))
;
690 for (auto &Count : this->Counts) {
691 bool Overflowed;
692 Count = SaturatingMultiply(Count, N, &Overflowed) / D;
693 if (Overflowed)
694 Warn(instrprof_error::counter_overflow);
695 }
696 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
697 scaleValueProfData(Kind, N, D, Warn);
698}
699
700// Map indirect call target name hash to name string.
701uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
702 InstrProfSymtab *SymTab) {
703 if (!SymTab)
704 return Value;
705
706 if (ValueKind == IPVK_IndirectCallTarget)
707 return SymTab->getFunctionHashFromAddress(Value);
708
709 return Value;
710}
711
712void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
713 InstrProfValueData *VData, uint32_t N,
714 InstrProfSymtab *ValueMap) {
715 for (uint32_t I = 0; I < N; I++) {
716 VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
717 }
718 std::vector<InstrProfValueSiteRecord> &ValueSites =
719 getOrCreateValueSitesForKind(ValueKind);
720 if (N == 0)
721 ValueSites.emplace_back();
722 else
723 ValueSites.emplace_back(VData, VData + N);
724}
725
726#define INSTR_PROF_COMMON_API_IMPL
727#include "llvm/ProfileData/InstrProfData.inc"
728
729/*!
730 * ValueProfRecordClosure Interface implementation for InstrProfRecord
731 * class. These C wrappers are used as adaptors so that C++ code can be
732 * invoked as callbacks.
733 */
734uint32_t getNumValueKindsInstrProf(const void *Record) {
735 return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
736}
737
738uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
739 return reinterpret_cast<const InstrProfRecord *>(Record)
740 ->getNumValueSites(VKind);
741}
742
743uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
744 return reinterpret_cast<const InstrProfRecord *>(Record)
745 ->getNumValueData(VKind);
746}
747
748uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
749 uint32_t S) {
750 return reinterpret_cast<const InstrProfRecord *>(R)
751 ->getNumValueDataForSite(VK, S);
752}
753
754void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
755 uint32_t K, uint32_t S) {
756 reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
757}
758
759ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
760 ValueProfData *VD =
761 (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
762 memset(VD, 0, TotalSizeInBytes);
763 return VD;
764}
765
766static ValueProfRecordClosure InstrProfRecordClosure = {
767 nullptr,
768 getNumValueKindsInstrProf,
769 getNumValueSitesInstrProf,
770 getNumValueDataInstrProf,
771 getNumValueDataForSiteInstrProf,
772 nullptr,
773 getValueForSiteInstrProf,
774 allocValueProfDataInstrProf};
775
776// Wrapper implementation using the closure mechanism.
777uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
778 auto Closure = InstrProfRecordClosure;
779 Closure.Record = &Record;
780 return getValueProfDataSize(&Closure);
781}
782
783// Wrapper implementation using the closure mechanism.
784std::unique_ptr<ValueProfData>
785ValueProfData::serializeFrom(const InstrProfRecord &Record) {
786 InstrProfRecordClosure.Record = &Record;
787
788 std::unique_ptr<ValueProfData> VPD(
789 serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
790 return VPD;
791}
792
793void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
794 InstrProfSymtab *SymTab) {
795 Record.reserveSites(Kind, NumValueSites);
796
797 InstrProfValueData *ValueData = getValueProfRecordValueData(this);
798 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
799 uint8_t ValueDataCount = this->SiteCountArray[VSite];
800 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
801 ValueData += ValueDataCount;
802 }
803}
804
805// For writing/serializing, Old is the host endianness, and New is
806// byte order intended on disk. For Reading/deserialization, Old
807// is the on-disk source endianness, and New is the host endianness.
808void ValueProfRecord::swapBytes(support::endianness Old,
809 support::endianness New) {
810 using namespace support;
811
812 if (Old == New)
813 return;
814
815 if (getHostEndianness() != Old) {
816 sys::swapByteOrder<uint32_t>(NumValueSites);
817 sys::swapByteOrder<uint32_t>(Kind);
818 }
819 uint32_t ND = getValueProfRecordNumValueData(this);
820 InstrProfValueData *VD = getValueProfRecordValueData(this);
821
822 // No need to swap byte array: SiteCountArrray.
823 for (uint32_t I = 0; I < ND; I++) {
824 sys::swapByteOrder<uint64_t>(VD[I].Value);
825 sys::swapByteOrder<uint64_t>(VD[I].Count);
826 }
827 if (getHostEndianness() == Old) {
828 sys::swapByteOrder<uint32_t>(NumValueSites);
829 sys::swapByteOrder<uint32_t>(Kind);
830 }
831}
832
833void ValueProfData::deserializeTo(InstrProfRecord &Record,
834 InstrProfSymtab *SymTab) {
835 if (NumValueKinds == 0)
836 return;
837
838 ValueProfRecord *VR = getFirstValueProfRecord(this);
839 for (uint32_t K = 0; K < NumValueKinds; K++) {
840 VR->deserializeTo(Record, SymTab);
841 VR = getValueProfRecordNext(VR);
842 }
843}
844
845template <class T>
846static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
847 using namespace support;
848
849 if (Orig == little)
850 return endian::readNext<T, little, unaligned>(D);
851 else
852 return endian::readNext<T, big, unaligned>(D);
853}
854
855static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
856 return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
857 ValueProfData());
858}
859
860Error ValueProfData::checkIntegrity() {
861 if (NumValueKinds > IPVK_Last + 1)
862 return make_error<InstrProfError>(instrprof_error::malformed);
863 // Total size needs to be mulltiple of quadword size.
864 if (TotalSize % sizeof(uint64_t))
865 return make_error<InstrProfError>(instrprof_error::malformed);
866
867 ValueProfRecord *VR = getFirstValueProfRecord(this);
868 for (uint32_t K = 0; K < this->NumValueKinds; K++) {
869 if (VR->Kind > IPVK_Last)
870 return make_error<InstrProfError>(instrprof_error::malformed);
871 VR = getValueProfRecordNext(VR);
872 if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
873 return make_error<InstrProfError>(instrprof_error::malformed);
874 }
875 return Error::success();
876}
877
878Expected<std::unique_ptr<ValueProfData>>
879ValueProfData::getValueProfData(const unsigned char *D,
880 const unsigned char *const BufferEnd,
881 support::endianness Endianness) {
882 using namespace support;
883
884 if (D + sizeof(ValueProfData) > BufferEnd)
885 return make_error<InstrProfError>(instrprof_error::truncated);
886
887 const unsigned char *Header = D;
888 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
889 if (D + TotalSize > BufferEnd)
890 return make_error<InstrProfError>(instrprof_error::too_large);
891
892 std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
893 memcpy(VPD.get(), D, TotalSize);
894 // Byte swap.
895 VPD->swapBytesToHost(Endianness);
896
897 Error E = VPD->checkIntegrity();
898 if (E)
899 return std::move(E);
900
901 return std::move(VPD);
902}
903
904void ValueProfData::swapBytesToHost(support::endianness Endianness) {
905 using namespace support;
906
907 if (Endianness == getHostEndianness())
908 return;
909
910 sys::swapByteOrder<uint32_t>(TotalSize);
911 sys::swapByteOrder<uint32_t>(NumValueKinds);
912
913 ValueProfRecord *VR = getFirstValueProfRecord(this);
914 for (uint32_t K = 0; K < NumValueKinds; K++) {
915 VR->swapBytes(Endianness, getHostEndianness());
916 VR = getValueProfRecordNext(VR);
917 }
918}
919
920void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
921 using namespace support;
922
923 if (Endianness == getHostEndianness())
924 return;
925
926 ValueProfRecord *VR = getFirstValueProfRecord(this);
927 for (uint32_t K = 0; K < NumValueKinds; K++) {
928 ValueProfRecord *NVR = getValueProfRecordNext(VR);
929 VR->swapBytes(getHostEndianness(), Endianness);
930 VR = NVR;
931 }
932 sys::swapByteOrder<uint32_t>(TotalSize);
933 sys::swapByteOrder<uint32_t>(NumValueKinds);
934}
935
936void annotateValueSite(Module &M, Instruction &Inst,
937 const InstrProfRecord &InstrProfR,
938 InstrProfValueKind ValueKind, uint32_t SiteIdx,
939 uint32_t MaxMDCount) {
940 uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
941 if (!NV)
942 return;
943
944 uint64_t Sum = 0;
945 std::unique_ptr<InstrProfValueData[]> VD =
946 InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
947
948 ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
949 annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
950}
951
952void annotateValueSite(Module &M, Instruction &Inst,
953 ArrayRef<InstrProfValueData> VDs,
954 uint64_t Sum, InstrProfValueKind ValueKind,
955 uint32_t MaxMDCount) {
956 LLVMContext &Ctx = M.getContext();
957 MDBuilder MDHelper(Ctx);
958 SmallVector<Metadata *, 3> Vals;
959 // Tag
960 Vals.push_back(MDHelper.createString("VP"));
961 // Value Kind
962 Vals.push_back(MDHelper.createConstant(
963 ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
964 // Total Count
965 Vals.push_back(
966 MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
967
968 // Value Profile Data
969 uint32_t MDCount = MaxMDCount;
970 for (auto &VD : VDs) {
971 Vals.push_back(MDHelper.createConstant(
972 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
973 Vals.push_back(MDHelper.createConstant(
974 ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
975 if (--MDCount == 0)
976 break;
977 }
978 Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
979}
980
981bool getValueProfDataFromInst(const Instruction &Inst,
982 InstrProfValueKind ValueKind,
983 uint32_t MaxNumValueData,
984 InstrProfValueData ValueData[],
985 uint32_t &ActualNumValueData, uint64_t &TotalC) {
986 MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
987 if (!MD)
988 return false;
989
990 unsigned NOps = MD->getNumOperands();
991
992 if (NOps < 5)
993 return false;
994
995 // Operand 0 is a string tag "VP":
996 MDString *Tag = cast<MDString>(MD->getOperand(0));
997 if (!Tag)
998 return false;
999
1000 if (!Tag->getString().equals("VP"))
1001 return false;
1002
1003 // Now check kind:
1004 ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1005 if (!KindInt)
1006 return false;
1007 if (KindInt->getZExtValue() != ValueKind)
1008 return false;
1009
1010 // Get total count
1011 ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1012 if (!TotalCInt)
1013 return false;
1014 TotalC = TotalCInt->getZExtValue();
1015
1016 ActualNumValueData = 0;
1017
1018 for (unsigned I = 3; I < NOps; I += 2) {
1019 if (ActualNumValueData >= MaxNumValueData)
1020 break;
1021 ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1022 ConstantInt *Count =
1023 mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1024 if (!Value || !Count)
1025 return false;
1026 ValueData[ActualNumValueData].Value = Value->getZExtValue();
1027 ValueData[ActualNumValueData].Count = Count->getZExtValue();
1028 ActualNumValueData++;
1029 }
1030 return true;
1031}
1032
1033MDNode *getPGOFuncNameMetadata(const Function &F) {
1034 return F.getMetadata(getPGOFuncNameMetadataName());
1035}
1036
1037void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
1038 // Only for internal linkage functions.
1039 if (PGOFuncName == F.getName())
1040 return;
1041 // Don't create duplicated meta-data.
1042 if (getPGOFuncNameMetadata(F))
1043 return;
1044 LLVMContext &C = F.getContext();
1045 MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1046 F.setMetadata(getPGOFuncNameMetadataName(), N);
1047}
1048
1049bool needsComdatForCounter(const Function &F, const Module &M) {
1050 if (F.hasComdat())
1051 return true;
1052
1053 if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1054 return false;
1055
1056 // See createPGOFuncNameVar for more details. To avoid link errors, profile
1057 // counters for function with available_externally linkage needs to be changed
1058 // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1059 // created. Without using comdat, duplicate entries won't be removed by the
1060 // linker leading to increased data segement size and raw profile size. Even
1061 // worse, since the referenced counter from profile per-function data object
1062 // will be resolved to the common strong definition, the profile counts for
1063 // available_externally functions will end up being duplicated in raw profile
1064 // data. This can result in distorted profile as the counts of those dups
1065 // will be accumulated by the profile merger.
1066 GlobalValue::LinkageTypes Linkage = F.getLinkage();
1067 if (Linkage != GlobalValue::ExternalWeakLinkage &&
1068 Linkage != GlobalValue::AvailableExternallyLinkage)
1069 return false;
1070
1071 return true;
1072}
1073
1074// Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1075bool isIRPGOFlagSet(const Module *M) {
1076 auto IRInstrVar =
1077 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version");
1078 if (!IRInstrVar || IRInstrVar->isDeclaration() ||
1079 IRInstrVar->hasLocalLinkage())
1080 return false;
1081
1082 // Check if the flag is set.
1083 if (!IRInstrVar->hasInitializer())
1084 return false;
1085
1086 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1087 if (!InitVal)
1088 return false;
1089 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF(0x1ULL << 56)) != 0;
1090}
1091
1092// Check if we can safely rename this Comdat function.
1093bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1094 if (F.getName().empty())
1095 return false;
1096 if (!needsComdatForCounter(F, *(F.getParent())))
1097 return false;
1098 // Unsafe to rename the address-taken function (which can be used in
1099 // function comparison).
1100 if (CheckAddressTaken && F.hasAddressTaken())
1101 return false;
1102 // Only safe to do if this function may be discarded if it is not used
1103 // in the compilation unit.
1104 if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1105 return false;
1106
1107 // For AvailableExternallyLinkage functions.
1108 if (!F.hasComdat()) {
1109 assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage)((F.getLinkage() == GlobalValue::AvailableExternallyLinkage) ?
static_cast<void> (0) : __assert_fail ("F.getLinkage() == GlobalValue::AvailableExternallyLinkage"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 1109, __PRETTY_FUNCTION__))
;
1110 return true;
1111 }
1112 return true;
1113}
1114
1115// FIXME: This is to be removed after switching to the new memop value
1116// profiling.
1117// Parse the value profile options.
1118void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
1119 int64_t &RangeLast) {
1120 static const int64_t DefaultMemOPSizeRangeStart = 0;
1121 static const int64_t DefaultMemOPSizeRangeLast = 8;
1122 RangeStart = DefaultMemOPSizeRangeStart;
1123 RangeLast = DefaultMemOPSizeRangeLast;
1124
1125 if (!MemOPSizeRange.empty()) {
1126 auto Pos = MemOPSizeRange.find(':');
1127 if (Pos != std::string::npos) {
1128 if (Pos > 0)
1129 MemOPSizeRange.substr(0, Pos).getAsInteger(10, RangeStart);
1130 if (Pos < MemOPSizeRange.size() - 1)
1131 MemOPSizeRange.substr(Pos + 1).getAsInteger(10, RangeLast);
1132 } else
1133 MemOPSizeRange.getAsInteger(10, RangeLast);
1134 }
1135 assert(RangeLast >= RangeStart)((RangeLast >= RangeStart) ? static_cast<void> (0) :
__assert_fail ("RangeLast >= RangeStart", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/ProfileData/InstrProf.cpp"
, 1135, __PRETTY_FUNCTION__))
;
1136}
1137
1138// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1139// aware this is an ir_level profile so it can set the version flag.
1140void createIRLevelProfileFlagVar(Module &M, bool IsCS,
1141 bool InstrEntryBBEnabled) {
1142 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version");
1143 Type *IntTy64 = Type::getInt64Ty(M.getContext());
1144 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION5 | VARIANT_MASK_IR_PROF(0x1ULL << 56));
1145 if (IsCS)
1146 ProfileVersion |= VARIANT_MASK_CSIR_PROF(0x1ULL << 57);
1147 if (InstrEntryBBEnabled)
1148 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY(0x1ULL << 58);
1149 auto IRLevelVersionVariable = new GlobalVariable(
1150 M, IntTy64, true, GlobalValue::WeakAnyLinkage,
1151 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
1152 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
1153 Triple TT(M.getTargetTriple());
1154 if (TT.supportsCOMDAT()) {
1155 IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage);
1156 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
1157 }
1158}
1159
1160// Create the variable for the profile file name.
1161void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1162 if (InstrProfileOutput.empty())
1163 return;
1164 Constant *ProfileNameConst =
1165 ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1166 GlobalVariable *ProfileNameVar = new GlobalVariable(
1167 M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1168 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename");
1169 Triple TT(M.getTargetTriple());
1170 if (TT.supportsCOMDAT()) {
1171 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
1172 ProfileNameVar->setComdat(M.getOrInsertComdat(
1173 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename")));
1174 }
1175}
1176
1177Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1178 const std::string &TestFilename,
1179 bool IsCS) {
1180 auto getProfileSum = [IsCS](const std::string &Filename,
1181 CountSumOrPercent &Sum) -> Error {
1182 auto ReaderOrErr = InstrProfReader::create(Filename);
1183 if (Error E = ReaderOrErr.takeError()) {
1184 return E;
1185 }
1186 auto Reader = std::move(ReaderOrErr.get());
1187 Reader->accumulateCounts(Sum, IsCS);
1188 return Error::success();
1189 };
1190 auto Ret = getProfileSum(BaseFilename, Base);
1191 if (Ret)
1192 return Ret;
1193 Ret = getProfileSum(TestFilename, Test);
1194 if (Ret)
1195 return Ret;
1196 this->BaseFilename = &BaseFilename;
1197 this->TestFilename = &TestFilename;
1198 Valid = true;
1199 return Error::success();
1200}
1201
1202void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) {
1203 Mismatch.NumEntries += 1;
1204 Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1205 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1206 if (Test.ValueCounts[I] >= 1.0f)
1207 Mismatch.ValueCounts[I] +=
1208 MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1209 }
1210}
1211
1212void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) {
1213 Unique.NumEntries += 1;
1214 Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1215 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1216 if (Test.ValueCounts[I] >= 1.0f)
1217 Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1218 }
1219}
1220
1221void OverlapStats::dump(raw_fd_ostream &OS) const {
1222 if (!Valid)
1223 return;
1224
1225 const char *EntryName =
1226 (Level == ProgramLevel ? "functions" : "edge counters");
1227 if (Level == ProgramLevel) {
1228 OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1229 << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1230 } else {
1231 OS << "Function level:\n"
1232 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1233 }
1234
1235 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1236 if (Mismatch.NumEntries)
1237 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1238 << "\n";
1239 if (Unique.NumEntries)
1240 OS << " # of " << EntryName
1241 << " only in test_profile: " << Unique.NumEntries << "\n";
1242
1243 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1244 << "\n";
1245 if (Mismatch.NumEntries)
1246 OS << " Mismatched count percentage (Edge): "
1247 << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1248 if (Unique.NumEntries)
1249 OS << " Percentage of Edge profile only in test_profile: "
1250 << format("%.3f%%", Unique.CountSum * 100) << "\n";
1251 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1252 << "\n"
1253 << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1254 << "\n";
1255
1256 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1257 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1258 continue;
1259 char ProfileKindName[20];
1260 switch (I) {
1261 case IPVK_IndirectCallTarget:
1262 strncpy(ProfileKindName, "IndirectCall", 19);
1263 break;
1264 case IPVK_MemOPSize:
1265 strncpy(ProfileKindName, "MemOP", 19);
1266 break;
1267 default:
1268 snprintf(ProfileKindName, 19, "VP[%d]", I);
1269 break;
1270 }
1271 OS << " " << ProfileKindName
1272 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1273 << "\n";
1274 if (Mismatch.NumEntries)
1275 OS << " Mismatched count percentage (" << ProfileKindName
1276 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1277 if (Unique.NumEntries)
1278 OS << " Percentage of " << ProfileKindName
1279 << " profile only in test_profile: "
1280 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1281 OS << " " << ProfileKindName
1282 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1283 << "\n"
1284 << " " << ProfileKindName
1285 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1286 << "\n";
1287 }
1288}
1289
1290} // end namespace llvm

/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/include/llvm/Support/LEB128.h

1//===- llvm/Support/LEB128.h - [SU]LEB128 utility functions -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares some utility functions for encoding SLEB128 and
10// ULEB128 values.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_LEB128_H
15#define LLVM_SUPPORT_LEB128_H
16
17#include "llvm/Support/raw_ostream.h"
18
19namespace llvm {
20
21/// Utility function to encode a SLEB128 value to an output stream. Returns
22/// the length in bytes of the encoded value.
23inline unsigned encodeSLEB128(int64_t Value, raw_ostream &OS,
24 unsigned PadTo = 0) {
25 bool More;
26 unsigned Count = 0;
27 do {
28 uint8_t Byte = Value & 0x7f;
29 // NOTE: this assumes that this signed shift is an arithmetic right shift.
30 Value >>= 7;
31 More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) ||
32 ((Value == -1) && ((Byte & 0x40) != 0))));
33 Count++;
34 if (More || Count < PadTo)
35 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
36 OS << char(Byte);
37 } while (More);
38
39 // Pad with 0x80 and emit a terminating byte at the end.
40 if (Count < PadTo) {
41 uint8_t PadValue = Value < 0 ? 0x7f : 0x00;
42 for (; Count < PadTo - 1; ++Count)
43 OS << char(PadValue | 0x80);
44 OS << char(PadValue);
45 Count++;
46 }
47 return Count;
48}
49
50/// Utility function to encode a SLEB128 value to a buffer. Returns
51/// the length in bytes of the encoded value.
52inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned PadTo = 0) {
53 uint8_t *orig_p = p;
54 unsigned Count = 0;
55 bool More;
56 do {
57 uint8_t Byte = Value & 0x7f;
58 // NOTE: this assumes that this signed shift is an arithmetic right shift.
59 Value >>= 7;
60 More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) ||
61 ((Value == -1) && ((Byte & 0x40) != 0))));
62 Count++;
63 if (More || Count < PadTo)
64 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
65 *p++ = Byte;
66 } while (More);
67
68 // Pad with 0x80 and emit a terminating byte at the end.
69 if (Count < PadTo) {
70 uint8_t PadValue = Value < 0 ? 0x7f : 0x00;
71 for (; Count < PadTo - 1; ++Count)
72 *p++ = (PadValue | 0x80);
73 *p++ = PadValue;
74 }
75 return (unsigned)(p - orig_p);
76}
77
78/// Utility function to encode a ULEB128 value to an output stream. Returns
79/// the length in bytes of the encoded value.
80inline unsigned encodeULEB128(uint64_t Value, raw_ostream &OS,
81 unsigned PadTo = 0) {
82 unsigned Count = 0;
83 do {
84 uint8_t Byte = Value & 0x7f;
85 Value >>= 7;
86 Count++;
87 if (Value != 0 || Count < PadTo)
88 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
89 OS << char(Byte);
90 } while (Value != 0);
91
92 // Pad with 0x80 and emit a null byte at the end.
93 if (Count < PadTo) {
94 for (; Count < PadTo - 1; ++Count)
95 OS << '\x80';
96 OS << '\x00';
97 Count++;
98 }
99 return Count;
100}
101
102/// Utility function to encode a ULEB128 value to a buffer. Returns
103/// the length in bytes of the encoded value.
104inline unsigned encodeULEB128(uint64_t Value, uint8_t *p,
105 unsigned PadTo = 0) {
106 uint8_t *orig_p = p;
107 unsigned Count = 0;
108 do {
109 uint8_t Byte = Value & 0x7f;
110 Value >>= 7;
111 Count++;
112 if (Value != 0 || Count < PadTo)
113 Byte |= 0x80; // Mark this byte to show that more bytes will follow.
114 *p++ = Byte;
115 } while (Value != 0);
116
117 // Pad with 0x80 and emit a null byte at the end.
118 if (Count < PadTo) {
119 for (; Count < PadTo - 1; ++Count)
120 *p++ = '\x80';
121 *p++ = '\x00';
122 }
123
124 return (unsigned)(p - orig_p);
125}
126
127/// Utility function to decode a ULEB128 value.
128inline uint64_t decodeULEB128(const uint8_t *p, unsigned *n = nullptr,
129 const uint8_t *end = nullptr,
130 const char **error = nullptr) {
131 const uint8_t *orig_p = p;
132 uint64_t Value = 0;
133 unsigned Shift = 0;
134 if (error
3.1
'error' is null
3.1
'error' is null
)
4
Taking false branch
135 *error = nullptr;
136 do {
137 if (p == end) {
5
Assuming 'p' is equal to 'end'
6
Taking true branch
138 if (error
6.1
'error' is null
6.1
'error' is null
)
7
Taking false branch
139 *error = "malformed uleb128, extends past end";
140 if (n
7.1
'n' is non-null
7.1
'n' is non-null
)
8
Taking true branch
141 *n = (unsigned)(p - orig_p);
142 return 0;
9
Returning zero
143 }
144 uint64_t Slice = *p & 0x7f;
145 if (Shift >= 64 || Slice << Shift >> Shift != Slice) {
146 if (error)
147 *error = "uleb128 too big for uint64";
148 if (n)
149 *n = (unsigned)(p - orig_p);
150 return 0;
151 }
152 Value += uint64_t(*p & 0x7f) << Shift;
153 Shift += 7;
154 } while (*p++ >= 128);
155 if (n)
156 *n = (unsigned)(p - orig_p);
157 return Value;
158}
159
160/// Utility function to decode a SLEB128 value.
161inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr,
162 const uint8_t *end = nullptr,
163 const char **error = nullptr) {
164 const uint8_t *orig_p = p;
165 int64_t Value = 0;
166 unsigned Shift = 0;
167 uint8_t Byte;
168 if (error)
169 *error = nullptr;
170 do {
171 if (p == end) {
172 if (error)
173 *error = "malformed sleb128, extends past end";
174 if (n)
175 *n = (unsigned)(p - orig_p);
176 return 0;
177 }
178 Byte = *p++;
179 Value |= (uint64_t(Byte & 0x7f) << Shift);
180 Shift += 7;
181 } while (Byte >= 128);
182 // Sign extend negative numbers if needed.
183 if (Shift < 64 && (Byte & 0x40))
184 Value |= (-1ULL) << Shift;
185 if (n)
186 *n = (unsigned)(p - orig_p);
187 return Value;
188}
189
190/// Utility function to get the size of the ULEB128-encoded value.
191extern unsigned getULEB128Size(uint64_t Value);
192
193/// Utility function to get the size of the SLEB128-encoded value.
194extern unsigned getSLEB128Size(int64_t Value);
195
196} // namespace llvm
197
198#endif // LLVM_SYSTEM_LEB128_H