File: | build/llvm-toolchain-snapshot-15~++20220314114050+9879c555f210/llvm/lib/ProfileData/InstrProf.cpp |
Warning: | line 537, column 24 Dereference of null pointer (loaded from variable 'P') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InstrProf.cpp - Instrumented profiling format support --------------===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // This file contains support for clang's instrumentation based PGO and | ||||
10 | // coverage. | ||||
11 | // | ||||
12 | //===----------------------------------------------------------------------===// | ||||
13 | |||||
14 | #include "llvm/ProfileData/InstrProf.h" | ||||
15 | #include "llvm/ADT/ArrayRef.h" | ||||
16 | #include "llvm/ADT/SmallString.h" | ||||
17 | #include "llvm/ADT/SmallVector.h" | ||||
18 | #include "llvm/ADT/StringExtras.h" | ||||
19 | #include "llvm/ADT/StringRef.h" | ||||
20 | #include "llvm/ADT/Triple.h" | ||||
21 | #include "llvm/Config/config.h" | ||||
22 | #include "llvm/IR/Constant.h" | ||||
23 | #include "llvm/IR/Constants.h" | ||||
24 | #include "llvm/IR/Function.h" | ||||
25 | #include "llvm/IR/GlobalValue.h" | ||||
26 | #include "llvm/IR/GlobalVariable.h" | ||||
27 | #include "llvm/IR/Instruction.h" | ||||
28 | #include "llvm/IR/LLVMContext.h" | ||||
29 | #include "llvm/IR/MDBuilder.h" | ||||
30 | #include "llvm/IR/Metadata.h" | ||||
31 | #include "llvm/IR/Module.h" | ||||
32 | #include "llvm/IR/Type.h" | ||||
33 | #include "llvm/ProfileData/InstrProfReader.h" | ||||
34 | #include "llvm/Support/Casting.h" | ||||
35 | #include "llvm/Support/CommandLine.h" | ||||
36 | #include "llvm/Support/Compiler.h" | ||||
37 | #include "llvm/Support/Compression.h" | ||||
38 | #include "llvm/Support/Endian.h" | ||||
39 | #include "llvm/Support/Error.h" | ||||
40 | #include "llvm/Support/ErrorHandling.h" | ||||
41 | #include "llvm/Support/LEB128.h" | ||||
42 | #include "llvm/Support/ManagedStatic.h" | ||||
43 | #include "llvm/Support/MathExtras.h" | ||||
44 | #include "llvm/Support/Path.h" | ||||
45 | #include "llvm/Support/SwapByteOrder.h" | ||||
46 | #include <algorithm> | ||||
47 | #include <cassert> | ||||
48 | #include <cstddef> | ||||
49 | #include <cstdint> | ||||
50 | #include <cstring> | ||||
51 | #include <memory> | ||||
52 | #include <string> | ||||
53 | #include <system_error> | ||||
54 | #include <type_traits> | ||||
55 | #include <utility> | ||||
56 | #include <vector> | ||||
57 | |||||
58 | using namespace llvm; | ||||
59 | |||||
60 | static cl::opt<bool> StaticFuncFullModulePrefix( | ||||
61 | "static-func-full-module-prefix", cl::init(true), cl::Hidden, | ||||
62 | cl::desc("Use full module build paths in the profile counter names for " | ||||
63 | "static functions.")); | ||||
64 | |||||
65 | // This option is tailored to users that have different top-level directory in | ||||
66 | // profile-gen and profile-use compilation. Users need to specific the number | ||||
67 | // of levels to strip. A value larger than the number of directories in the | ||||
68 | // source file will strip all the directory names and only leave the basename. | ||||
69 | // | ||||
70 | // Note current ThinLTO module importing for the indirect-calls assumes | ||||
71 | // the source directory name not being stripped. A non-zero option value here | ||||
72 | // can potentially prevent some inter-module indirect-call-promotions. | ||||
73 | static cl::opt<unsigned> StaticFuncStripDirNamePrefix( | ||||
74 | "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, | ||||
75 | cl::desc("Strip specified level of directory name from source path in " | ||||
76 | "the profile counter name for static functions.")); | ||||
77 | |||||
78 | static std::string getInstrProfErrString(instrprof_error Err, | ||||
79 | const std::string &ErrMsg = "") { | ||||
80 | std::string Msg; | ||||
81 | raw_string_ostream OS(Msg); | ||||
82 | |||||
83 | switch (Err) { | ||||
84 | case instrprof_error::success: | ||||
85 | OS << "success"; | ||||
86 | break; | ||||
87 | case instrprof_error::eof: | ||||
88 | OS << "end of File"; | ||||
89 | break; | ||||
90 | case instrprof_error::unrecognized_format: | ||||
91 | OS << "unrecognized instrumentation profile encoding format"; | ||||
92 | break; | ||||
93 | case instrprof_error::bad_magic: | ||||
94 | OS << "invalid instrumentation profile data (bad magic)"; | ||||
95 | break; | ||||
96 | case instrprof_error::bad_header: | ||||
97 | OS << "invalid instrumentation profile data (file header is corrupt)"; | ||||
98 | break; | ||||
99 | case instrprof_error::unsupported_version: | ||||
100 | OS << "unsupported instrumentation profile format version"; | ||||
101 | break; | ||||
102 | case instrprof_error::unsupported_hash_type: | ||||
103 | OS << "unsupported instrumentation profile hash type"; | ||||
104 | break; | ||||
105 | case instrprof_error::too_large: | ||||
106 | OS << "too much profile data"; | ||||
107 | break; | ||||
108 | case instrprof_error::truncated: | ||||
109 | OS << "truncated profile data"; | ||||
110 | break; | ||||
111 | case instrprof_error::malformed: | ||||
112 | OS << "malformed instrumentation profile data"; | ||||
113 | break; | ||||
114 | case instrprof_error::missing_debug_info_for_correlation: | ||||
115 | OS << "debug info for correlation is required"; | ||||
116 | break; | ||||
117 | case instrprof_error::unexpected_debug_info_for_correlation: | ||||
118 | OS << "debug info for correlation is not necessary"; | ||||
119 | break; | ||||
120 | case instrprof_error::unable_to_correlate_profile: | ||||
121 | OS << "unable to correlate profile"; | ||||
122 | break; | ||||
123 | case instrprof_error::invalid_prof: | ||||
124 | OS << "invalid profile created. Please file a bug " | ||||
125 | "at: " BUG_REPORT_URL"https://github.com/llvm/llvm-project/issues/" | ||||
126 | " and include the profraw files that caused this error."; | ||||
127 | break; | ||||
128 | case instrprof_error::unknown_function: | ||||
129 | OS << "no profile data available for function"; | ||||
130 | break; | ||||
131 | case instrprof_error::hash_mismatch: | ||||
132 | OS << "function control flow change detected (hash mismatch)"; | ||||
133 | break; | ||||
134 | case instrprof_error::count_mismatch: | ||||
135 | OS << "function basic block count change detected (counter mismatch)"; | ||||
136 | break; | ||||
137 | case instrprof_error::counter_overflow: | ||||
138 | OS << "counter overflow"; | ||||
139 | break; | ||||
140 | case instrprof_error::value_site_count_mismatch: | ||||
141 | OS << "function value site count change detected (counter mismatch)"; | ||||
142 | break; | ||||
143 | case instrprof_error::compress_failed: | ||||
144 | OS << "failed to compress data (zlib)"; | ||||
145 | break; | ||||
146 | case instrprof_error::uncompress_failed: | ||||
147 | OS << "failed to uncompress data (zlib)"; | ||||
148 | break; | ||||
149 | case instrprof_error::empty_raw_profile: | ||||
150 | OS << "empty raw profile file"; | ||||
151 | break; | ||||
152 | case instrprof_error::zlib_unavailable: | ||||
153 | OS << "profile uses zlib compression but the profile reader was built " | ||||
154 | "without zlib support"; | ||||
155 | break; | ||||
156 | } | ||||
157 | |||||
158 | // If optional error message is not empty, append it to the message. | ||||
159 | if (!ErrMsg.empty()) | ||||
160 | OS << ": " << ErrMsg; | ||||
161 | |||||
162 | return OS.str(); | ||||
163 | } | ||||
164 | |||||
165 | namespace { | ||||
166 | |||||
167 | // FIXME: This class is only here to support the transition to llvm::Error. It | ||||
168 | // will be removed once this transition is complete. Clients should prefer to | ||||
169 | // deal with the Error value directly, rather than converting to error_code. | ||||
170 | class InstrProfErrorCategoryType : public std::error_category { | ||||
171 | const char *name() const noexcept override { return "llvm.instrprof"; } | ||||
172 | |||||
173 | std::string message(int IE) const override { | ||||
174 | return getInstrProfErrString(static_cast<instrprof_error>(IE)); | ||||
175 | } | ||||
176 | }; | ||||
177 | |||||
178 | } // end anonymous namespace | ||||
179 | |||||
180 | static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory; | ||||
181 | |||||
182 | const std::error_category &llvm::instrprof_category() { | ||||
183 | return *ErrorCategory; | ||||
184 | } | ||||
185 | |||||
186 | namespace { | ||||
187 | |||||
188 | const char *InstrProfSectNameCommon[] = { | ||||
189 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ | ||||
190 | SectNameCommon, | ||||
191 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
192 | }; | ||||
193 | |||||
194 | const char *InstrProfSectNameCoff[] = { | ||||
195 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ | ||||
196 | SectNameCoff, | ||||
197 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
198 | }; | ||||
199 | |||||
200 | const char *InstrProfSectNamePrefix[] = { | ||||
201 | #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \ | ||||
202 | Prefix, | ||||
203 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
204 | }; | ||||
205 | |||||
206 | } // namespace | ||||
207 | |||||
208 | namespace llvm { | ||||
209 | |||||
210 | cl::opt<bool> DoInstrProfNameCompression( | ||||
211 | "enable-name-compression", | ||||
212 | cl::desc("Enable name/filename string compression"), cl::init(true)); | ||||
213 | |||||
214 | std::string getInstrProfSectionName(InstrProfSectKind IPSK, | ||||
215 | Triple::ObjectFormatType OF, | ||||
216 | bool AddSegmentInfo) { | ||||
217 | std::string SectName; | ||||
218 | |||||
219 | if (OF == Triple::MachO && AddSegmentInfo) | ||||
220 | SectName = InstrProfSectNamePrefix[IPSK]; | ||||
221 | |||||
222 | if (OF == Triple::COFF) | ||||
223 | SectName += InstrProfSectNameCoff[IPSK]; | ||||
224 | else | ||||
225 | SectName += InstrProfSectNameCommon[IPSK]; | ||||
226 | |||||
227 | if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo) | ||||
228 | SectName += ",regular,live_support"; | ||||
229 | |||||
230 | return SectName; | ||||
231 | } | ||||
232 | |||||
233 | void SoftInstrProfErrors::addError(instrprof_error IE) { | ||||
234 | if (IE == instrprof_error::success) | ||||
235 | return; | ||||
236 | |||||
237 | if (FirstError == instrprof_error::success) | ||||
238 | FirstError = IE; | ||||
239 | |||||
240 | switch (IE) { | ||||
241 | case instrprof_error::hash_mismatch: | ||||
242 | ++NumHashMismatches; | ||||
243 | break; | ||||
244 | case instrprof_error::count_mismatch: | ||||
245 | ++NumCountMismatches; | ||||
246 | break; | ||||
247 | case instrprof_error::counter_overflow: | ||||
248 | ++NumCounterOverflows; | ||||
249 | break; | ||||
250 | case instrprof_error::value_site_count_mismatch: | ||||
251 | ++NumValueSiteCountMismatches; | ||||
252 | break; | ||||
253 | default: | ||||
254 | llvm_unreachable("Not a soft error")::llvm::llvm_unreachable_internal("Not a soft error", "llvm/lib/ProfileData/InstrProf.cpp" , 254); | ||||
255 | } | ||||
256 | } | ||||
257 | |||||
258 | std::string InstrProfError::message() const { | ||||
259 | return getInstrProfErrString(Err, Msg); | ||||
260 | } | ||||
261 | |||||
262 | char InstrProfError::ID = 0; | ||||
263 | |||||
264 | std::string getPGOFuncName(StringRef RawFuncName, | ||||
265 | GlobalValue::LinkageTypes Linkage, | ||||
266 | StringRef FileName, | ||||
267 | uint64_t Version LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__))) { | ||||
268 | return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); | ||||
269 | } | ||||
270 | |||||
271 | // Strip NumPrefix level of directory name from PathNameStr. If the number of | ||||
272 | // directory separators is less than NumPrefix, strip all the directories and | ||||
273 | // leave base file name only. | ||||
274 | static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { | ||||
275 | uint32_t Count = NumPrefix; | ||||
276 | uint32_t Pos = 0, LastPos = 0; | ||||
277 | for (auto & CI : PathNameStr) { | ||||
278 | ++Pos; | ||||
279 | if (llvm::sys::path::is_separator(CI)) { | ||||
280 | LastPos = Pos; | ||||
281 | --Count; | ||||
282 | } | ||||
283 | if (Count == 0) | ||||
284 | break; | ||||
285 | } | ||||
286 | return PathNameStr.substr(LastPos); | ||||
287 | } | ||||
288 | |||||
289 | // Return the PGOFuncName. This function has some special handling when called | ||||
290 | // in LTO optimization. The following only applies when calling in LTO passes | ||||
291 | // (when \c InLTO is true): LTO's internalization privatizes many global linkage | ||||
292 | // symbols. This happens after value profile annotation, but those internal | ||||
293 | // linkage functions should not have a source prefix. | ||||
294 | // Additionally, for ThinLTO mode, exported internal functions are promoted | ||||
295 | // and renamed. We need to ensure that the original internal PGO name is | ||||
296 | // used when computing the GUID that is compared against the profiled GUIDs. | ||||
297 | // To differentiate compiler generated internal symbols from original ones, | ||||
298 | // PGOFuncName meta data are created and attached to the original internal | ||||
299 | // symbols in the value profile annotation step | ||||
300 | // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta | ||||
301 | // data, its original linkage must be non-internal. | ||||
302 | std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { | ||||
303 | if (!InLTO) { | ||||
304 | StringRef FileName(F.getParent()->getSourceFileName()); | ||||
305 | uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1; | ||||
306 | if (StripLevel < StaticFuncStripDirNamePrefix) | ||||
307 | StripLevel = StaticFuncStripDirNamePrefix; | ||||
308 | if (StripLevel) | ||||
309 | FileName = stripDirPrefix(FileName, StripLevel); | ||||
310 | return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version); | ||||
311 | } | ||||
312 | |||||
313 | // In LTO mode (when InLTO is true), first check if there is a meta data. | ||||
314 | if (MDNode *MD = getPGOFuncNameMetadata(F)) { | ||||
315 | StringRef S = cast<MDString>(MD->getOperand(0))->getString(); | ||||
316 | return S.str(); | ||||
317 | } | ||||
318 | |||||
319 | // If there is no meta data, the function must be a global before the value | ||||
320 | // profile annotation pass. Its current linkage may be internal if it is | ||||
321 | // internalized in LTO mode. | ||||
322 | return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); | ||||
323 | } | ||||
324 | |||||
325 | StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { | ||||
326 | if (FileName.empty()) | ||||
327 | return PGOFuncName; | ||||
328 | // Drop the file name including ':'. See also getPGOFuncName. | ||||
329 | if (PGOFuncName.startswith(FileName)) | ||||
330 | PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1); | ||||
331 | return PGOFuncName; | ||||
332 | } | ||||
333 | |||||
334 | // \p FuncName is the string used as profile lookup key for the function. A | ||||
335 | // symbol is created to hold the name. Return the legalized symbol name. | ||||
336 | std::string getPGOFuncNameVarName(StringRef FuncName, | ||||
337 | GlobalValue::LinkageTypes Linkage) { | ||||
338 | std::string VarName = std::string(getInstrProfNameVarPrefix()); | ||||
339 | VarName += FuncName; | ||||
340 | |||||
341 | if (!GlobalValue::isLocalLinkage(Linkage)) | ||||
342 | return VarName; | ||||
343 | |||||
344 | // Now fix up illegal chars in local VarName that may upset the assembler. | ||||
345 | const char *InvalidChars = "-:<>/\"'"; | ||||
346 | size_t found = VarName.find_first_of(InvalidChars); | ||||
347 | while (found != std::string::npos) { | ||||
348 | VarName[found] = '_'; | ||||
349 | found = VarName.find_first_of(InvalidChars, found + 1); | ||||
350 | } | ||||
351 | return VarName; | ||||
352 | } | ||||
353 | |||||
354 | GlobalVariable *createPGOFuncNameVar(Module &M, | ||||
355 | GlobalValue::LinkageTypes Linkage, | ||||
356 | StringRef PGOFuncName) { | ||||
357 | // We generally want to match the function's linkage, but available_externally | ||||
358 | // and extern_weak both have the wrong semantics, and anything that doesn't | ||||
359 | // need to link across compilation units doesn't need to be visible at all. | ||||
360 | if (Linkage == GlobalValue::ExternalWeakLinkage) | ||||
361 | Linkage = GlobalValue::LinkOnceAnyLinkage; | ||||
362 | else if (Linkage == GlobalValue::AvailableExternallyLinkage) | ||||
363 | Linkage = GlobalValue::LinkOnceODRLinkage; | ||||
364 | else if (Linkage == GlobalValue::InternalLinkage || | ||||
365 | Linkage == GlobalValue::ExternalLinkage) | ||||
366 | Linkage = GlobalValue::PrivateLinkage; | ||||
367 | |||||
368 | auto *Value = | ||||
369 | ConstantDataArray::getString(M.getContext(), PGOFuncName, false); | ||||
370 | auto FuncNameVar = | ||||
371 | new GlobalVariable(M, Value->getType(), true, Linkage, Value, | ||||
372 | getPGOFuncNameVarName(PGOFuncName, Linkage)); | ||||
373 | |||||
374 | // Hide the symbol so that we correctly get a copy for each executable. | ||||
375 | if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) | ||||
376 | FuncNameVar->setVisibility(GlobalValue::HiddenVisibility); | ||||
377 | |||||
378 | return FuncNameVar; | ||||
379 | } | ||||
380 | |||||
381 | GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) { | ||||
382 | return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName); | ||||
383 | } | ||||
384 | |||||
385 | Error InstrProfSymtab::create(Module &M, bool InLTO) { | ||||
386 | for (Function &F : M) { | ||||
387 | // Function may not have a name: like using asm("") to overwrite the name. | ||||
388 | // Ignore in this case. | ||||
389 | if (!F.hasName()) | ||||
390 | continue; | ||||
391 | const std::string &PGOFuncName = getPGOFuncName(F, InLTO); | ||||
392 | if (Error E = addFuncName(PGOFuncName)) | ||||
393 | return E; | ||||
394 | MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); | ||||
395 | // In ThinLTO, local function may have been promoted to global and have | ||||
396 | // suffix ".llvm." added to the function name. We need to add the | ||||
397 | // stripped function name to the symbol table so that we can find a match | ||||
398 | // from profile. | ||||
399 | // | ||||
400 | // We may have other suffixes similar as ".llvm." which are needed to | ||||
401 | // be stripped before the matching, but ".__uniq." suffix which is used | ||||
402 | // to differentiate internal linkage functions in different modules | ||||
403 | // should be kept. Now this is the only suffix with the pattern ".xxx" | ||||
404 | // which is kept before matching. | ||||
405 | const std::string UniqSuffix = ".__uniq."; | ||||
406 | auto pos = PGOFuncName.find(UniqSuffix); | ||||
407 | // Search '.' after ".__uniq." if ".__uniq." exists, otherwise | ||||
408 | // search '.' from the beginning. | ||||
409 | if (pos != std::string::npos) | ||||
410 | pos += UniqSuffix.length(); | ||||
411 | else | ||||
412 | pos = 0; | ||||
413 | pos = PGOFuncName.find('.', pos); | ||||
414 | if (pos != std::string::npos && pos != 0) { | ||||
415 | const std::string &OtherFuncName = PGOFuncName.substr(0, pos); | ||||
416 | if (Error E = addFuncName(OtherFuncName)) | ||||
417 | return E; | ||||
418 | MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); | ||||
419 | } | ||||
420 | } | ||||
421 | Sorted = false; | ||||
422 | finalizeSymtab(); | ||||
423 | return Error::success(); | ||||
424 | } | ||||
425 | |||||
426 | uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { | ||||
427 | finalizeSymtab(); | ||||
428 | auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) { | ||||
429 | return A.first < Address; | ||||
430 | }); | ||||
431 | // Raw function pointer collected by value profiler may be from | ||||
432 | // external functions that are not instrumented. They won't have | ||||
433 | // mapping data to be used by the deserializer. Force the value to | ||||
434 | // be 0 in this case. | ||||
435 | if (It != AddrToMD5Map.end() && It->first == Address) | ||||
436 | return (uint64_t)It->second; | ||||
437 | return 0; | ||||
438 | } | ||||
439 | |||||
440 | Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs, | ||||
441 | bool doCompression, std::string &Result) { | ||||
442 | assert(!NameStrs.empty() && "No name data to emit")(static_cast <bool> (!NameStrs.empty() && "No name data to emit" ) ? void (0) : __assert_fail ("!NameStrs.empty() && \"No name data to emit\"" , "llvm/lib/ProfileData/InstrProf.cpp", 442, __extension__ __PRETTY_FUNCTION__ )); | ||||
443 | |||||
444 | uint8_t Header[16], *P = Header; | ||||
445 | std::string UncompressedNameStrings = | ||||
446 | join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator()); | ||||
447 | |||||
448 | assert(StringRef(UncompressedNameStrings)(static_cast <bool> (StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)") ? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\"" , "llvm/lib/ProfileData/InstrProf.cpp", 450, __extension__ __PRETTY_FUNCTION__ )) | ||||
449 | .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&(static_cast <bool> (StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)") ? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\"" , "llvm/lib/ProfileData/InstrProf.cpp", 450, __extension__ __PRETTY_FUNCTION__ )) | ||||
450 | "PGO name is invalid (contains separator token)")(static_cast <bool> (StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && "PGO name is invalid (contains separator token)") ? void (0) : __assert_fail ("StringRef(UncompressedNameStrings) .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) && \"PGO name is invalid (contains separator token)\"" , "llvm/lib/ProfileData/InstrProf.cpp", 450, __extension__ __PRETTY_FUNCTION__ )); | ||||
451 | |||||
452 | unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P); | ||||
453 | P += EncLen; | ||||
454 | |||||
455 | auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) { | ||||
456 | EncLen = encodeULEB128(CompressedLen, P); | ||||
457 | P += EncLen; | ||||
458 | char *HeaderStr = reinterpret_cast<char *>(&Header[0]); | ||||
459 | unsigned HeaderLen = P - &Header[0]; | ||||
460 | Result.append(HeaderStr, HeaderLen); | ||||
461 | Result += InputStr; | ||||
462 | return Error::success(); | ||||
463 | }; | ||||
464 | |||||
465 | if (!doCompression) { | ||||
466 | return WriteStringToResult(0, UncompressedNameStrings); | ||||
467 | } | ||||
468 | |||||
469 | SmallString<128> CompressedNameStrings; | ||||
470 | Error E = zlib::compress(StringRef(UncompressedNameStrings), | ||||
471 | CompressedNameStrings, zlib::BestSizeCompression); | ||||
472 | if (E) { | ||||
473 | consumeError(std::move(E)); | ||||
474 | return make_error<InstrProfError>(instrprof_error::compress_failed); | ||||
475 | } | ||||
476 | |||||
477 | return WriteStringToResult(CompressedNameStrings.size(), | ||||
478 | CompressedNameStrings); | ||||
479 | } | ||||
480 | |||||
481 | StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) { | ||||
482 | auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer()); | ||||
483 | StringRef NameStr = | ||||
484 | Arr->isCString() ? Arr->getAsCString() : Arr->getAsString(); | ||||
485 | return NameStr; | ||||
486 | } | ||||
487 | |||||
488 | Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, | ||||
489 | std::string &Result, bool doCompression) { | ||||
490 | std::vector<std::string> NameStrs; | ||||
491 | for (auto *NameVar : NameVars) { | ||||
492 | NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar))); | ||||
493 | } | ||||
494 | return collectPGOFuncNameStrings( | ||||
495 | NameStrs, zlib::isAvailable() && doCompression, Result); | ||||
496 | } | ||||
497 | |||||
498 | Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) { | ||||
499 | const uint8_t *P = NameStrings.bytes_begin(); | ||||
500 | const uint8_t *EndP = NameStrings.bytes_end(); | ||||
501 | while (P < EndP) { | ||||
| |||||
502 | uint32_t N; | ||||
503 | uint64_t UncompressedSize = decodeULEB128(P, &N); | ||||
504 | P += N; | ||||
505 | uint64_t CompressedSize = decodeULEB128(P, &N); | ||||
506 | P += N; | ||||
507 | bool isCompressed = (CompressedSize != 0); | ||||
508 | SmallString<128> UncompressedNameStrings; | ||||
509 | StringRef NameStrings; | ||||
510 | if (isCompressed
| ||||
511 | if (!llvm::zlib::isAvailable()) | ||||
512 | return make_error<InstrProfError>(instrprof_error::zlib_unavailable); | ||||
513 | |||||
514 | StringRef CompressedNameStrings(reinterpret_cast<const char *>(P), | ||||
515 | CompressedSize); | ||||
516 | if (Error E = | ||||
517 | zlib::uncompress(CompressedNameStrings, UncompressedNameStrings, | ||||
518 | UncompressedSize)) { | ||||
519 | consumeError(std::move(E)); | ||||
520 | return make_error<InstrProfError>(instrprof_error::uncompress_failed); | ||||
521 | } | ||||
522 | P += CompressedSize; | ||||
523 | NameStrings = StringRef(UncompressedNameStrings.data(), | ||||
524 | UncompressedNameStrings.size()); | ||||
525 | } else { | ||||
526 | NameStrings = | ||||
527 | StringRef(reinterpret_cast<const char *>(P), UncompressedSize); | ||||
528 | P += UncompressedSize; | ||||
529 | } | ||||
530 | // Now parse the name strings. | ||||
531 | SmallVector<StringRef, 0> Names; | ||||
532 | NameStrings.split(Names, getInstrProfNameSeparator()); | ||||
533 | for (StringRef &Name : Names) | ||||
534 | if (Error E = Symtab.addFuncName(Name)) | ||||
535 | return E; | ||||
536 | |||||
537 | while (P
| ||||
| |||||
538 | P++; | ||||
539 | } | ||||
540 | return Error::success(); | ||||
541 | } | ||||
542 | |||||
543 | void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { | ||||
544 | uint64_t FuncSum = 0; | ||||
545 | Sum.NumEntries += Counts.size(); | ||||
546 | for (uint64_t Count : Counts) | ||||
547 | FuncSum += Count; | ||||
548 | Sum.CountSum += FuncSum; | ||||
549 | |||||
550 | for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) { | ||||
551 | uint64_t KindSum = 0; | ||||
552 | uint32_t NumValueSites = getNumValueSites(VK); | ||||
553 | for (size_t I = 0; I < NumValueSites; ++I) { | ||||
554 | uint32_t NV = getNumValueDataForSite(VK, I); | ||||
555 | std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I); | ||||
556 | for (uint32_t V = 0; V < NV; V++) | ||||
557 | KindSum += VD[V].Count; | ||||
558 | } | ||||
559 | Sum.ValueCounts[VK] += KindSum; | ||||
560 | } | ||||
561 | } | ||||
562 | |||||
563 | void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input, | ||||
564 | uint32_t ValueKind, | ||||
565 | OverlapStats &Overlap, | ||||
566 | OverlapStats &FuncLevelOverlap) { | ||||
567 | this->sortByTargetValues(); | ||||
568 | Input.sortByTargetValues(); | ||||
569 | double Score = 0.0f, FuncLevelScore = 0.0f; | ||||
570 | auto I = ValueData.begin(); | ||||
571 | auto IE = ValueData.end(); | ||||
572 | auto J = Input.ValueData.begin(); | ||||
573 | auto JE = Input.ValueData.end(); | ||||
574 | while (I != IE && J != JE) { | ||||
575 | if (I->Value == J->Value) { | ||||
576 | Score += OverlapStats::score(I->Count, J->Count, | ||||
577 | Overlap.Base.ValueCounts[ValueKind], | ||||
578 | Overlap.Test.ValueCounts[ValueKind]); | ||||
579 | FuncLevelScore += OverlapStats::score( | ||||
580 | I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind], | ||||
581 | FuncLevelOverlap.Test.ValueCounts[ValueKind]); | ||||
582 | ++I; | ||||
583 | } else if (I->Value < J->Value) { | ||||
584 | ++I; | ||||
585 | continue; | ||||
586 | } | ||||
587 | ++J; | ||||
588 | } | ||||
589 | Overlap.Overlap.ValueCounts[ValueKind] += Score; | ||||
590 | FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore; | ||||
591 | } | ||||
592 | |||||
593 | // Return false on mismatch. | ||||
594 | void InstrProfRecord::overlapValueProfData(uint32_t ValueKind, | ||||
595 | InstrProfRecord &Other, | ||||
596 | OverlapStats &Overlap, | ||||
597 | OverlapStats &FuncLevelOverlap) { | ||||
598 | uint32_t ThisNumValueSites = getNumValueSites(ValueKind); | ||||
599 | assert(ThisNumValueSites == Other.getNumValueSites(ValueKind))(static_cast <bool> (ThisNumValueSites == Other.getNumValueSites (ValueKind)) ? void (0) : __assert_fail ("ThisNumValueSites == Other.getNumValueSites(ValueKind)" , "llvm/lib/ProfileData/InstrProf.cpp", 599, __extension__ __PRETTY_FUNCTION__ )); | ||||
600 | if (!ThisNumValueSites) | ||||
601 | return; | ||||
602 | |||||
603 | std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = | ||||
604 | getOrCreateValueSitesForKind(ValueKind); | ||||
605 | MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = | ||||
606 | Other.getValueSitesForKind(ValueKind); | ||||
607 | for (uint32_t I = 0; I < ThisNumValueSites; I++) | ||||
608 | ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap, | ||||
609 | FuncLevelOverlap); | ||||
610 | } | ||||
611 | |||||
612 | void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap, | ||||
613 | OverlapStats &FuncLevelOverlap, | ||||
614 | uint64_t ValueCutoff) { | ||||
615 | // FuncLevel CountSum for other should already computed and nonzero. | ||||
616 | assert(FuncLevelOverlap.Test.CountSum >= 1.0f)(static_cast <bool> (FuncLevelOverlap.Test.CountSum >= 1.0f) ? void (0) : __assert_fail ("FuncLevelOverlap.Test.CountSum >= 1.0f" , "llvm/lib/ProfileData/InstrProf.cpp", 616, __extension__ __PRETTY_FUNCTION__ )); | ||||
617 | accumulateCounts(FuncLevelOverlap.Base); | ||||
618 | bool Mismatch = (Counts.size() != Other.Counts.size()); | ||||
619 | |||||
620 | // Check if the value profiles mismatch. | ||||
621 | if (!Mismatch) { | ||||
622 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { | ||||
623 | uint32_t ThisNumValueSites = getNumValueSites(Kind); | ||||
624 | uint32_t OtherNumValueSites = Other.getNumValueSites(Kind); | ||||
625 | if (ThisNumValueSites != OtherNumValueSites) { | ||||
626 | Mismatch = true; | ||||
627 | break; | ||||
628 | } | ||||
629 | } | ||||
630 | } | ||||
631 | if (Mismatch) { | ||||
632 | Overlap.addOneMismatch(FuncLevelOverlap.Test); | ||||
633 | return; | ||||
634 | } | ||||
635 | |||||
636 | // Compute overlap for value counts. | ||||
637 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) | ||||
638 | overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap); | ||||
639 | |||||
640 | double Score = 0.0; | ||||
641 | uint64_t MaxCount = 0; | ||||
642 | // Compute overlap for edge counts. | ||||
643 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { | ||||
644 | Score += OverlapStats::score(Counts[I], Other.Counts[I], | ||||
645 | Overlap.Base.CountSum, Overlap.Test.CountSum); | ||||
646 | MaxCount = std::max(Other.Counts[I], MaxCount); | ||||
647 | } | ||||
648 | Overlap.Overlap.CountSum += Score; | ||||
649 | Overlap.Overlap.NumEntries += 1; | ||||
650 | |||||
651 | if (MaxCount >= ValueCutoff) { | ||||
652 | double FuncScore = 0.0; | ||||
653 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) | ||||
654 | FuncScore += OverlapStats::score(Counts[I], Other.Counts[I], | ||||
655 | FuncLevelOverlap.Base.CountSum, | ||||
656 | FuncLevelOverlap.Test.CountSum); | ||||
657 | FuncLevelOverlap.Overlap.CountSum = FuncScore; | ||||
658 | FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size(); | ||||
659 | FuncLevelOverlap.Valid = true; | ||||
660 | } | ||||
661 | } | ||||
662 | |||||
663 | void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, | ||||
664 | uint64_t Weight, | ||||
665 | function_ref<void(instrprof_error)> Warn) { | ||||
666 | this->sortByTargetValues(); | ||||
667 | Input.sortByTargetValues(); | ||||
668 | auto I = ValueData.begin(); | ||||
669 | auto IE = ValueData.end(); | ||||
670 | for (const InstrProfValueData &J : Input.ValueData) { | ||||
671 | while (I != IE && I->Value < J.Value) | ||||
672 | ++I; | ||||
673 | if (I != IE && I->Value == J.Value) { | ||||
674 | bool Overflowed; | ||||
675 | I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed); | ||||
676 | if (Overflowed) | ||||
677 | Warn(instrprof_error::counter_overflow); | ||||
678 | ++I; | ||||
679 | continue; | ||||
680 | } | ||||
681 | ValueData.insert(I, J); | ||||
682 | } | ||||
683 | } | ||||
684 | |||||
685 | void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D, | ||||
686 | function_ref<void(instrprof_error)> Warn) { | ||||
687 | for (InstrProfValueData &I : ValueData) { | ||||
688 | bool Overflowed; | ||||
689 | I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D; | ||||
690 | if (Overflowed) | ||||
691 | Warn(instrprof_error::counter_overflow); | ||||
692 | } | ||||
693 | } | ||||
694 | |||||
695 | // Merge Value Profile data from Src record to this record for ValueKind. | ||||
696 | // Scale merged value counts by \p Weight. | ||||
697 | void InstrProfRecord::mergeValueProfData( | ||||
698 | uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight, | ||||
699 | function_ref<void(instrprof_error)> Warn) { | ||||
700 | uint32_t ThisNumValueSites = getNumValueSites(ValueKind); | ||||
701 | uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); | ||||
702 | if (ThisNumValueSites != OtherNumValueSites) { | ||||
703 | Warn(instrprof_error::value_site_count_mismatch); | ||||
704 | return; | ||||
705 | } | ||||
706 | if (!ThisNumValueSites) | ||||
707 | return; | ||||
708 | std::vector<InstrProfValueSiteRecord> &ThisSiteRecords = | ||||
709 | getOrCreateValueSitesForKind(ValueKind); | ||||
710 | MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords = | ||||
711 | Src.getValueSitesForKind(ValueKind); | ||||
712 | for (uint32_t I = 0; I < ThisNumValueSites; I++) | ||||
713 | ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn); | ||||
714 | } | ||||
715 | |||||
716 | void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, | ||||
717 | function_ref<void(instrprof_error)> Warn) { | ||||
718 | // If the number of counters doesn't match we either have bad data | ||||
719 | // or a hash collision. | ||||
720 | if (Counts.size() != Other.Counts.size()) { | ||||
721 | Warn(instrprof_error::count_mismatch); | ||||
722 | return; | ||||
723 | } | ||||
724 | |||||
725 | for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) { | ||||
726 | bool Overflowed; | ||||
727 | Counts[I] = | ||||
728 | SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); | ||||
729 | if (Overflowed) | ||||
730 | Warn(instrprof_error::counter_overflow); | ||||
731 | } | ||||
732 | |||||
733 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) | ||||
734 | mergeValueProfData(Kind, Other, Weight, Warn); | ||||
735 | } | ||||
736 | |||||
737 | void InstrProfRecord::scaleValueProfData( | ||||
738 | uint32_t ValueKind, uint64_t N, uint64_t D, | ||||
739 | function_ref<void(instrprof_error)> Warn) { | ||||
740 | for (auto &R : getValueSitesForKind(ValueKind)) | ||||
741 | R.scale(N, D, Warn); | ||||
742 | } | ||||
743 | |||||
744 | void InstrProfRecord::scale(uint64_t N, uint64_t D, | ||||
745 | function_ref<void(instrprof_error)> Warn) { | ||||
746 | assert(D != 0 && "D cannot be 0")(static_cast <bool> (D != 0 && "D cannot be 0") ? void (0) : __assert_fail ("D != 0 && \"D cannot be 0\"" , "llvm/lib/ProfileData/InstrProf.cpp", 746, __extension__ __PRETTY_FUNCTION__ )); | ||||
747 | for (auto &Count : this->Counts) { | ||||
748 | bool Overflowed; | ||||
749 | Count = SaturatingMultiply(Count, N, &Overflowed) / D; | ||||
750 | if (Overflowed) | ||||
751 | Warn(instrprof_error::counter_overflow); | ||||
752 | } | ||||
753 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) | ||||
754 | scaleValueProfData(Kind, N, D, Warn); | ||||
755 | } | ||||
756 | |||||
757 | // Map indirect call target name hash to name string. | ||||
758 | uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, | ||||
759 | InstrProfSymtab *SymTab) { | ||||
760 | if (!SymTab) | ||||
761 | return Value; | ||||
762 | |||||
763 | if (ValueKind == IPVK_IndirectCallTarget) | ||||
764 | return SymTab->getFunctionHashFromAddress(Value); | ||||
765 | |||||
766 | return Value; | ||||
767 | } | ||||
768 | |||||
769 | void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, | ||||
770 | InstrProfValueData *VData, uint32_t N, | ||||
771 | InstrProfSymtab *ValueMap) { | ||||
772 | for (uint32_t I = 0; I < N; I++) { | ||||
773 | VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap); | ||||
774 | } | ||||
775 | std::vector<InstrProfValueSiteRecord> &ValueSites = | ||||
776 | getOrCreateValueSitesForKind(ValueKind); | ||||
777 | if (N == 0) | ||||
778 | ValueSites.emplace_back(); | ||||
779 | else | ||||
780 | ValueSites.emplace_back(VData, VData + N); | ||||
781 | } | ||||
782 | |||||
783 | #define INSTR_PROF_COMMON_API_IMPL | ||||
784 | #include "llvm/ProfileData/InstrProfData.inc" | ||||
785 | |||||
786 | /*! | ||||
787 | * ValueProfRecordClosure Interface implementation for InstrProfRecord | ||||
788 | * class. These C wrappers are used as adaptors so that C++ code can be | ||||
789 | * invoked as callbacks. | ||||
790 | */ | ||||
791 | uint32_t getNumValueKindsInstrProf(const void *Record) { | ||||
792 | return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds(); | ||||
793 | } | ||||
794 | |||||
795 | uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) { | ||||
796 | return reinterpret_cast<const InstrProfRecord *>(Record) | ||||
797 | ->getNumValueSites(VKind); | ||||
798 | } | ||||
799 | |||||
800 | uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) { | ||||
801 | return reinterpret_cast<const InstrProfRecord *>(Record) | ||||
802 | ->getNumValueData(VKind); | ||||
803 | } | ||||
804 | |||||
805 | uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, | ||||
806 | uint32_t S) { | ||||
807 | return reinterpret_cast<const InstrProfRecord *>(R) | ||||
808 | ->getNumValueDataForSite(VK, S); | ||||
809 | } | ||||
810 | |||||
811 | void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, | ||||
812 | uint32_t K, uint32_t S) { | ||||
813 | reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S); | ||||
814 | } | ||||
815 | |||||
816 | ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { | ||||
817 | ValueProfData *VD = | ||||
818 | (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData()); | ||||
819 | memset(VD, 0, TotalSizeInBytes); | ||||
820 | return VD; | ||||
821 | } | ||||
822 | |||||
823 | static ValueProfRecordClosure InstrProfRecordClosure = { | ||||
824 | nullptr, | ||||
825 | getNumValueKindsInstrProf, | ||||
826 | getNumValueSitesInstrProf, | ||||
827 | getNumValueDataInstrProf, | ||||
828 | getNumValueDataForSiteInstrProf, | ||||
829 | nullptr, | ||||
830 | getValueForSiteInstrProf, | ||||
831 | allocValueProfDataInstrProf}; | ||||
832 | |||||
833 | // Wrapper implementation using the closure mechanism. | ||||
834 | uint32_t ValueProfData::getSize(const InstrProfRecord &Record) { | ||||
835 | auto Closure = InstrProfRecordClosure; | ||||
836 | Closure.Record = &Record; | ||||
837 | return getValueProfDataSize(&Closure); | ||||
838 | } | ||||
839 | |||||
840 | // Wrapper implementation using the closure mechanism. | ||||
841 | std::unique_ptr<ValueProfData> | ||||
842 | ValueProfData::serializeFrom(const InstrProfRecord &Record) { | ||||
843 | InstrProfRecordClosure.Record = &Record; | ||||
844 | |||||
845 | std::unique_ptr<ValueProfData> VPD( | ||||
846 | serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr)); | ||||
847 | return VPD; | ||||
848 | } | ||||
849 | |||||
850 | void ValueProfRecord::deserializeTo(InstrProfRecord &Record, | ||||
851 | InstrProfSymtab *SymTab) { | ||||
852 | Record.reserveSites(Kind, NumValueSites); | ||||
853 | |||||
854 | InstrProfValueData *ValueData = getValueProfRecordValueData(this); | ||||
855 | for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) { | ||||
856 | uint8_t ValueDataCount = this->SiteCountArray[VSite]; | ||||
857 | Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab); | ||||
858 | ValueData += ValueDataCount; | ||||
859 | } | ||||
860 | } | ||||
861 | |||||
862 | // For writing/serializing, Old is the host endianness, and New is | ||||
863 | // byte order intended on disk. For Reading/deserialization, Old | ||||
864 | // is the on-disk source endianness, and New is the host endianness. | ||||
865 | void ValueProfRecord::swapBytes(support::endianness Old, | ||||
866 | support::endianness New) { | ||||
867 | using namespace support; | ||||
868 | |||||
869 | if (Old == New) | ||||
870 | return; | ||||
871 | |||||
872 | if (getHostEndianness() != Old) { | ||||
873 | sys::swapByteOrder<uint32_t>(NumValueSites); | ||||
874 | sys::swapByteOrder<uint32_t>(Kind); | ||||
875 | } | ||||
876 | uint32_t ND = getValueProfRecordNumValueData(this); | ||||
877 | InstrProfValueData *VD = getValueProfRecordValueData(this); | ||||
878 | |||||
879 | // No need to swap byte array: SiteCountArrray. | ||||
880 | for (uint32_t I = 0; I < ND; I++) { | ||||
881 | sys::swapByteOrder<uint64_t>(VD[I].Value); | ||||
882 | sys::swapByteOrder<uint64_t>(VD[I].Count); | ||||
883 | } | ||||
884 | if (getHostEndianness() == Old) { | ||||
885 | sys::swapByteOrder<uint32_t>(NumValueSites); | ||||
886 | sys::swapByteOrder<uint32_t>(Kind); | ||||
887 | } | ||||
888 | } | ||||
889 | |||||
890 | void ValueProfData::deserializeTo(InstrProfRecord &Record, | ||||
891 | InstrProfSymtab *SymTab) { | ||||
892 | if (NumValueKinds == 0) | ||||
893 | return; | ||||
894 | |||||
895 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
896 | for (uint32_t K = 0; K < NumValueKinds; K++) { | ||||
897 | VR->deserializeTo(Record, SymTab); | ||||
898 | VR = getValueProfRecordNext(VR); | ||||
899 | } | ||||
900 | } | ||||
901 | |||||
902 | template <class T> | ||||
903 | static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) { | ||||
904 | using namespace support; | ||||
905 | |||||
906 | if (Orig == little) | ||||
907 | return endian::readNext<T, little, unaligned>(D); | ||||
908 | else | ||||
909 | return endian::readNext<T, big, unaligned>(D); | ||||
910 | } | ||||
911 | |||||
912 | static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) { | ||||
913 | return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize)) | ||||
914 | ValueProfData()); | ||||
915 | } | ||||
916 | |||||
917 | Error ValueProfData::checkIntegrity() { | ||||
918 | if (NumValueKinds > IPVK_Last + 1) | ||||
919 | return make_error<InstrProfError>( | ||||
920 | instrprof_error::malformed, "number of value profile kinds is invalid"); | ||||
921 | // Total size needs to be multiple of quadword size. | ||||
922 | if (TotalSize % sizeof(uint64_t)) | ||||
923 | return make_error<InstrProfError>( | ||||
924 | instrprof_error::malformed, "total size is not multiples of quardword"); | ||||
925 | |||||
926 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
927 | for (uint32_t K = 0; K < this->NumValueKinds; K++) { | ||||
928 | if (VR->Kind > IPVK_Last) | ||||
929 | return make_error<InstrProfError>(instrprof_error::malformed, | ||||
930 | "value kind is invalid"); | ||||
931 | VR = getValueProfRecordNext(VR); | ||||
932 | if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize) | ||||
933 | return make_error<InstrProfError>( | ||||
934 | instrprof_error::malformed, | ||||
935 | "value profile address is greater than total size"); | ||||
936 | } | ||||
937 | return Error::success(); | ||||
938 | } | ||||
939 | |||||
940 | Expected<std::unique_ptr<ValueProfData>> | ||||
941 | ValueProfData::getValueProfData(const unsigned char *D, | ||||
942 | const unsigned char *const BufferEnd, | ||||
943 | support::endianness Endianness) { | ||||
944 | using namespace support; | ||||
945 | |||||
946 | if (D + sizeof(ValueProfData) > BufferEnd) | ||||
947 | return make_error<InstrProfError>(instrprof_error::truncated); | ||||
948 | |||||
949 | const unsigned char *Header = D; | ||||
950 | uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness); | ||||
951 | if (D + TotalSize > BufferEnd) | ||||
952 | return make_error<InstrProfError>(instrprof_error::too_large); | ||||
953 | |||||
954 | std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize); | ||||
955 | memcpy(VPD.get(), D, TotalSize); | ||||
956 | // Byte swap. | ||||
957 | VPD->swapBytesToHost(Endianness); | ||||
958 | |||||
959 | Error E = VPD->checkIntegrity(); | ||||
960 | if (E) | ||||
961 | return std::move(E); | ||||
962 | |||||
963 | return std::move(VPD); | ||||
964 | } | ||||
965 | |||||
966 | void ValueProfData::swapBytesToHost(support::endianness Endianness) { | ||||
967 | using namespace support; | ||||
968 | |||||
969 | if (Endianness == getHostEndianness()) | ||||
970 | return; | ||||
971 | |||||
972 | sys::swapByteOrder<uint32_t>(TotalSize); | ||||
973 | sys::swapByteOrder<uint32_t>(NumValueKinds); | ||||
974 | |||||
975 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
976 | for (uint32_t K = 0; K < NumValueKinds; K++) { | ||||
977 | VR->swapBytes(Endianness, getHostEndianness()); | ||||
978 | VR = getValueProfRecordNext(VR); | ||||
979 | } | ||||
980 | } | ||||
981 | |||||
982 | void ValueProfData::swapBytesFromHost(support::endianness Endianness) { | ||||
983 | using namespace support; | ||||
984 | |||||
985 | if (Endianness == getHostEndianness()) | ||||
986 | return; | ||||
987 | |||||
988 | ValueProfRecord *VR = getFirstValueProfRecord(this); | ||||
989 | for (uint32_t K = 0; K < NumValueKinds; K++) { | ||||
990 | ValueProfRecord *NVR = getValueProfRecordNext(VR); | ||||
991 | VR->swapBytes(getHostEndianness(), Endianness); | ||||
992 | VR = NVR; | ||||
993 | } | ||||
994 | sys::swapByteOrder<uint32_t>(TotalSize); | ||||
995 | sys::swapByteOrder<uint32_t>(NumValueKinds); | ||||
996 | } | ||||
997 | |||||
998 | void annotateValueSite(Module &M, Instruction &Inst, | ||||
999 | const InstrProfRecord &InstrProfR, | ||||
1000 | InstrProfValueKind ValueKind, uint32_t SiteIdx, | ||||
1001 | uint32_t MaxMDCount) { | ||||
1002 | uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx); | ||||
1003 | if (!NV) | ||||
1004 | return; | ||||
1005 | |||||
1006 | uint64_t Sum = 0; | ||||
1007 | std::unique_ptr<InstrProfValueData[]> VD = | ||||
1008 | InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum); | ||||
1009 | |||||
1010 | ArrayRef<InstrProfValueData> VDs(VD.get(), NV); | ||||
1011 | annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount); | ||||
1012 | } | ||||
1013 | |||||
1014 | void annotateValueSite(Module &M, Instruction &Inst, | ||||
1015 | ArrayRef<InstrProfValueData> VDs, | ||||
1016 | uint64_t Sum, InstrProfValueKind ValueKind, | ||||
1017 | uint32_t MaxMDCount) { | ||||
1018 | LLVMContext &Ctx = M.getContext(); | ||||
1019 | MDBuilder MDHelper(Ctx); | ||||
1020 | SmallVector<Metadata *, 3> Vals; | ||||
1021 | // Tag | ||||
1022 | Vals.push_back(MDHelper.createString("VP")); | ||||
1023 | // Value Kind | ||||
1024 | Vals.push_back(MDHelper.createConstant( | ||||
1025 | ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind))); | ||||
1026 | // Total Count | ||||
1027 | Vals.push_back( | ||||
1028 | MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum))); | ||||
1029 | |||||
1030 | // Value Profile Data | ||||
1031 | uint32_t MDCount = MaxMDCount; | ||||
1032 | for (auto &VD : VDs) { | ||||
1033 | Vals.push_back(MDHelper.createConstant( | ||||
1034 | ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value))); | ||||
1035 | Vals.push_back(MDHelper.createConstant( | ||||
1036 | ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count))); | ||||
1037 | if (--MDCount == 0) | ||||
1038 | break; | ||||
1039 | } | ||||
1040 | Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals)); | ||||
1041 | } | ||||
1042 | |||||
1043 | bool getValueProfDataFromInst(const Instruction &Inst, | ||||
1044 | InstrProfValueKind ValueKind, | ||||
1045 | uint32_t MaxNumValueData, | ||||
1046 | InstrProfValueData ValueData[], | ||||
1047 | uint32_t &ActualNumValueData, uint64_t &TotalC, | ||||
1048 | bool GetNoICPValue) { | ||||
1049 | MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof); | ||||
1050 | if (!MD) | ||||
1051 | return false; | ||||
1052 | |||||
1053 | unsigned NOps = MD->getNumOperands(); | ||||
1054 | |||||
1055 | if (NOps < 5) | ||||
1056 | return false; | ||||
1057 | |||||
1058 | // Operand 0 is a string tag "VP": | ||||
1059 | MDString *Tag = cast<MDString>(MD->getOperand(0)); | ||||
1060 | if (!Tag) | ||||
1061 | return false; | ||||
1062 | |||||
1063 | if (!Tag->getString().equals("VP")) | ||||
1064 | return false; | ||||
1065 | |||||
1066 | // Now check kind: | ||||
1067 | ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1)); | ||||
1068 | if (!KindInt) | ||||
1069 | return false; | ||||
1070 | if (KindInt->getZExtValue() != ValueKind) | ||||
1071 | return false; | ||||
1072 | |||||
1073 | // Get total count | ||||
1074 | ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); | ||||
1075 | if (!TotalCInt) | ||||
1076 | return false; | ||||
1077 | TotalC = TotalCInt->getZExtValue(); | ||||
1078 | |||||
1079 | ActualNumValueData = 0; | ||||
1080 | |||||
1081 | for (unsigned I = 3; I < NOps; I += 2) { | ||||
1082 | if (ActualNumValueData >= MaxNumValueData) | ||||
1083 | break; | ||||
1084 | ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I)); | ||||
1085 | ConstantInt *Count = | ||||
1086 | mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1)); | ||||
1087 | if (!Value || !Count) | ||||
1088 | return false; | ||||
1089 | uint64_t CntValue = Count->getZExtValue(); | ||||
1090 | if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM)) | ||||
1091 | continue; | ||||
1092 | ValueData[ActualNumValueData].Value = Value->getZExtValue(); | ||||
1093 | ValueData[ActualNumValueData].Count = CntValue; | ||||
1094 | ActualNumValueData++; | ||||
1095 | } | ||||
1096 | return true; | ||||
1097 | } | ||||
1098 | |||||
1099 | MDNode *getPGOFuncNameMetadata(const Function &F) { | ||||
1100 | return F.getMetadata(getPGOFuncNameMetadataName()); | ||||
1101 | } | ||||
1102 | |||||
1103 | void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { | ||||
1104 | // Only for internal linkage functions. | ||||
1105 | if (PGOFuncName == F.getName()) | ||||
1106 | return; | ||||
1107 | // Don't create duplicated meta-data. | ||||
1108 | if (getPGOFuncNameMetadata(F)) | ||||
1109 | return; | ||||
1110 | LLVMContext &C = F.getContext(); | ||||
1111 | MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName)); | ||||
1112 | F.setMetadata(getPGOFuncNameMetadataName(), N); | ||||
1113 | } | ||||
1114 | |||||
1115 | bool needsComdatForCounter(const Function &F, const Module &M) { | ||||
1116 | if (F.hasComdat()) | ||||
1117 | return true; | ||||
1118 | |||||
1119 | if (!Triple(M.getTargetTriple()).supportsCOMDAT()) | ||||
1120 | return false; | ||||
1121 | |||||
1122 | // See createPGOFuncNameVar for more details. To avoid link errors, profile | ||||
1123 | // counters for function with available_externally linkage needs to be changed | ||||
1124 | // to linkonce linkage. On ELF based systems, this leads to weak symbols to be | ||||
1125 | // created. Without using comdat, duplicate entries won't be removed by the | ||||
1126 | // linker leading to increased data segement size and raw profile size. Even | ||||
1127 | // worse, since the referenced counter from profile per-function data object | ||||
1128 | // will be resolved to the common strong definition, the profile counts for | ||||
1129 | // available_externally functions will end up being duplicated in raw profile | ||||
1130 | // data. This can result in distorted profile as the counts of those dups | ||||
1131 | // will be accumulated by the profile merger. | ||||
1132 | GlobalValue::LinkageTypes Linkage = F.getLinkage(); | ||||
1133 | if (Linkage != GlobalValue::ExternalWeakLinkage && | ||||
1134 | Linkage != GlobalValue::AvailableExternallyLinkage) | ||||
1135 | return false; | ||||
1136 | |||||
1137 | return true; | ||||
1138 | } | ||||
1139 | |||||
1140 | // Check if INSTR_PROF_RAW_VERSION_VAR is defined. | ||||
1141 | bool isIRPGOFlagSet(const Module *M) { | ||||
1142 | auto IRInstrVar = | ||||
1143 | M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version"); | ||||
1144 | if (!IRInstrVar || IRInstrVar->hasLocalLinkage()) | ||||
1145 | return false; | ||||
1146 | |||||
1147 | // For CSPGO+LTO, this variable might be marked as non-prevailing and we only | ||||
1148 | // have the decl. | ||||
1149 | if (IRInstrVar->isDeclaration()) | ||||
1150 | return true; | ||||
1151 | |||||
1152 | // Check if the flag is set. | ||||
1153 | if (!IRInstrVar->hasInitializer()) | ||||
1154 | return false; | ||||
1155 | |||||
1156 | auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer()); | ||||
1157 | if (!InitVal) | ||||
1158 | return false; | ||||
1159 | return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF(0x1ULL << 56)) != 0; | ||||
1160 | } | ||||
1161 | |||||
1162 | // Check if we can safely rename this Comdat function. | ||||
1163 | bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) { | ||||
1164 | if (F.getName().empty()) | ||||
1165 | return false; | ||||
1166 | if (!needsComdatForCounter(F, *(F.getParent()))) | ||||
1167 | return false; | ||||
1168 | // Unsafe to rename the address-taken function (which can be used in | ||||
1169 | // function comparison). | ||||
1170 | if (CheckAddressTaken && F.hasAddressTaken()) | ||||
1171 | return false; | ||||
1172 | // Only safe to do if this function may be discarded if it is not used | ||||
1173 | // in the compilation unit. | ||||
1174 | if (!GlobalValue::isDiscardableIfUnused(F.getLinkage())) | ||||
1175 | return false; | ||||
1176 | |||||
1177 | // For AvailableExternallyLinkage functions. | ||||
1178 | if (!F.hasComdat()) { | ||||
1179 | assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage)(static_cast <bool> (F.getLinkage() == GlobalValue::AvailableExternallyLinkage ) ? void (0) : __assert_fail ("F.getLinkage() == GlobalValue::AvailableExternallyLinkage" , "llvm/lib/ProfileData/InstrProf.cpp", 1179, __extension__ __PRETTY_FUNCTION__ )); | ||||
1180 | return true; | ||||
1181 | } | ||||
1182 | return true; | ||||
1183 | } | ||||
1184 | |||||
1185 | // Create the variable for the profile file name. | ||||
1186 | void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) { | ||||
1187 | if (InstrProfileOutput.empty()) | ||||
1188 | return; | ||||
1189 | Constant *ProfileNameConst = | ||||
1190 | ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true); | ||||
1191 | GlobalVariable *ProfileNameVar = new GlobalVariable( | ||||
1192 | M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage, | ||||
1193 | ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename"); | ||||
1194 | Triple TT(M.getTargetTriple()); | ||||
1195 | if (TT.supportsCOMDAT()) { | ||||
1196 | ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage); | ||||
1197 | ProfileNameVar->setComdat(M.getOrInsertComdat( | ||||
1198 | StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)"__llvm_profile_filename"))); | ||||
1199 | } | ||||
1200 | } | ||||
1201 | |||||
1202 | Error OverlapStats::accumulateCounts(const std::string &BaseFilename, | ||||
1203 | const std::string &TestFilename, | ||||
1204 | bool IsCS) { | ||||
1205 | auto getProfileSum = [IsCS](const std::string &Filename, | ||||
1206 | CountSumOrPercent &Sum) -> Error { | ||||
1207 | auto ReaderOrErr = InstrProfReader::create(Filename); | ||||
1208 | if (Error E = ReaderOrErr.takeError()) { | ||||
1209 | return E; | ||||
1210 | } | ||||
1211 | auto Reader = std::move(ReaderOrErr.get()); | ||||
1212 | Reader->accumulateCounts(Sum, IsCS); | ||||
1213 | return Error::success(); | ||||
1214 | }; | ||||
1215 | auto Ret = getProfileSum(BaseFilename, Base); | ||||
1216 | if (Ret) | ||||
1217 | return Ret; | ||||
1218 | Ret = getProfileSum(TestFilename, Test); | ||||
1219 | if (Ret) | ||||
1220 | return Ret; | ||||
1221 | this->BaseFilename = &BaseFilename; | ||||
1222 | this->TestFilename = &TestFilename; | ||||
1223 | Valid = true; | ||||
1224 | return Error::success(); | ||||
1225 | } | ||||
1226 | |||||
1227 | void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) { | ||||
1228 | Mismatch.NumEntries += 1; | ||||
1229 | Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum; | ||||
1230 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { | ||||
1231 | if (Test.ValueCounts[I] >= 1.0f) | ||||
1232 | Mismatch.ValueCounts[I] += | ||||
1233 | MismatchFunc.ValueCounts[I] / Test.ValueCounts[I]; | ||||
1234 | } | ||||
1235 | } | ||||
1236 | |||||
1237 | void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) { | ||||
1238 | Unique.NumEntries += 1; | ||||
1239 | Unique.CountSum += UniqueFunc.CountSum / Test.CountSum; | ||||
1240 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { | ||||
1241 | if (Test.ValueCounts[I] >= 1.0f) | ||||
1242 | Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I]; | ||||
1243 | } | ||||
1244 | } | ||||
1245 | |||||
1246 | void OverlapStats::dump(raw_fd_ostream &OS) const { | ||||
1247 | if (!Valid) | ||||
1248 | return; | ||||
1249 | |||||
1250 | const char *EntryName = | ||||
1251 | (Level == ProgramLevel ? "functions" : "edge counters"); | ||||
1252 | if (Level == ProgramLevel) { | ||||
1253 | OS << "Profile overlap infomation for base_profile: " << *BaseFilename | ||||
1254 | << " and test_profile: " << *TestFilename << "\nProgram level:\n"; | ||||
1255 | } else { | ||||
1256 | OS << "Function level:\n" | ||||
1257 | << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n"; | ||||
1258 | } | ||||
1259 | |||||
1260 | OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n"; | ||||
1261 | if (Mismatch.NumEntries) | ||||
1262 | OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries | ||||
1263 | << "\n"; | ||||
1264 | if (Unique.NumEntries) | ||||
1265 | OS << " # of " << EntryName | ||||
1266 | << " only in test_profile: " << Unique.NumEntries << "\n"; | ||||
1267 | |||||
1268 | OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100) | ||||
1269 | << "\n"; | ||||
1270 | if (Mismatch.NumEntries) | ||||
1271 | OS << " Mismatched count percentage (Edge): " | ||||
1272 | << format("%.3f%%", Mismatch.CountSum * 100) << "\n"; | ||||
1273 | if (Unique.NumEntries) | ||||
1274 | OS << " Percentage of Edge profile only in test_profile: " | ||||
1275 | << format("%.3f%%", Unique.CountSum * 100) << "\n"; | ||||
1276 | OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum) | ||||
1277 | << "\n" | ||||
1278 | << " Edge profile test count sum: " << format("%.0f", Test.CountSum) | ||||
1279 | << "\n"; | ||||
1280 | |||||
1281 | for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { | ||||
1282 | if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) | ||||
1283 | continue; | ||||
1284 | char ProfileKindName[20]; | ||||
1285 | switch (I) { | ||||
1286 | case IPVK_IndirectCallTarget: | ||||
1287 | strncpy(ProfileKindName, "IndirectCall", 19); | ||||
1288 | break; | ||||
1289 | case IPVK_MemOPSize: | ||||
1290 | strncpy(ProfileKindName, "MemOP", 19); | ||||
1291 | break; | ||||
1292 | default: | ||||
1293 | snprintf(ProfileKindName, 19, "VP[%d]", I); | ||||
1294 | break; | ||||
1295 | } | ||||
1296 | OS << " " << ProfileKindName | ||||
1297 | << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100) | ||||
1298 | << "\n"; | ||||
1299 | if (Mismatch.NumEntries) | ||||
1300 | OS << " Mismatched count percentage (" << ProfileKindName | ||||
1301 | << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n"; | ||||
1302 | if (Unique.NumEntries) | ||||
1303 | OS << " Percentage of " << ProfileKindName | ||||
1304 | << " profile only in test_profile: " | ||||
1305 | << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n"; | ||||
1306 | OS << " " << ProfileKindName | ||||
1307 | << " profile base count sum: " << format("%.0f", Base.ValueCounts[I]) | ||||
1308 | << "\n" | ||||
1309 | << " " << ProfileKindName | ||||
1310 | << " profile test count sum: " << format("%.0f", Test.ValueCounts[I]) | ||||
1311 | << "\n"; | ||||
1312 | } | ||||
1313 | } | ||||
1314 | |||||
1315 | namespace IndexedInstrProf { | ||||
1316 | // A C++14 compatible version of the offsetof macro. | ||||
1317 | template <typename T1, typename T2> | ||||
1318 | inline size_t constexpr offsetOf(T1 T2::*Member) { | ||||
1319 | constexpr T2 Object{}; | ||||
1320 | return size_t(&(Object.*Member)) - size_t(&Object); | ||||
1321 | } | ||||
1322 | |||||
1323 | static inline uint64_t read(const unsigned char *Buffer, size_t Offset) { | ||||
1324 | return *reinterpret_cast<const uint64_t *>(Buffer + Offset); | ||||
1325 | } | ||||
1326 | |||||
1327 | uint64_t Header::formatVersion() const { | ||||
1328 | using namespace support; | ||||
1329 | return endian::byte_swap<uint64_t, little>(Version); | ||||
1330 | } | ||||
1331 | |||||
1332 | Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) { | ||||
1333 | using namespace support; | ||||
1334 | static_assert(std::is_standard_layout<Header>::value, | ||||
1335 | "The header should be standard layout type since we use offset " | ||||
1336 | "of fields to read."); | ||||
1337 | Header H; | ||||
1338 | |||||
1339 | H.Magic = read(Buffer, offsetOf(&Header::Magic)); | ||||
1340 | // Check the magic number. | ||||
1341 | uint64_t Magic = endian::byte_swap<uint64_t, little>(H.Magic); | ||||
1342 | if (Magic != IndexedInstrProf::Magic) | ||||
1343 | return make_error<InstrProfError>(instrprof_error::bad_magic); | ||||
1344 | |||||
1345 | // Read the version. | ||||
1346 | H.Version = read(Buffer, offsetOf(&Header::Version)); | ||||
1347 | if (GET_VERSION(H.formatVersion())((H.formatVersion()) & ~0xff00000000000000ULL) > | ||||
1348 | IndexedInstrProf::ProfVersion::CurrentVersion) | ||||
1349 | return make_error<InstrProfError>(instrprof_error::unsupported_version); | ||||
1350 | |||||
1351 | switch (GET_VERSION(H.formatVersion())((H.formatVersion()) & ~0xff00000000000000ULL)) { | ||||
1352 | // When a new field is added in the header add a case statement here to | ||||
1353 | // populate it. | ||||
1354 | static_assert( | ||||
1355 | IndexedInstrProf::ProfVersion::CurrentVersion == Version8, | ||||
1356 | "Please update the reading code below if a new field has been added, " | ||||
1357 | "if not add a case statement to fall through to the latest version."); | ||||
1358 | case 8ull: | ||||
1359 | H.MemProfOffset = read(Buffer, offsetOf(&Header::MemProfOffset)); | ||||
1360 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||
1361 | default: // Version7 (when the backwards compatible header was introduced). | ||||
1362 | H.HashType = read(Buffer, offsetOf(&Header::HashType)); | ||||
1363 | H.HashOffset = read(Buffer, offsetOf(&Header::HashOffset)); | ||||
1364 | } | ||||
1365 | |||||
1366 | return H; | ||||
1367 | } | ||||
1368 | |||||
1369 | size_t Header::size() const { | ||||
1370 | switch (GET_VERSION(formatVersion())((formatVersion()) & ~0xff00000000000000ULL)) { | ||||
1371 | // When a new field is added to the header add a case statement here to | ||||
1372 | // compute the size as offset of the new field + size of the new field. This | ||||
1373 | // relies on the field being added to the end of the list. | ||||
1374 | static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version8, | ||||
1375 | "Please update the size computation below if a new field has " | ||||
1376 | "been added to the header, if not add a case statement to " | ||||
1377 | "fall through to the latest version."); | ||||
1378 | case 8ull: | ||||
1379 | return offsetOf(&Header::MemProfOffset) + sizeof(Header::MemProfOffset); | ||||
1380 | default: // Version7 (when the backwards compatible header was introduced). | ||||
1381 | return offsetOf(&Header::HashOffset) + sizeof(Header::HashOffset); | ||||
1382 | } | ||||
1383 | } | ||||
1384 | |||||
1385 | } // namespace IndexedInstrProf | ||||
1386 | |||||
1387 | } // end namespace llvm |
1 | //===- llvm/Support/LEB128.h - [SU]LEB128 utility functions -----*- C++ -*-===// | ||||
2 | // | ||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||
6 | // | ||||
7 | //===----------------------------------------------------------------------===// | ||||
8 | // | ||||
9 | // This file declares some utility functions for encoding SLEB128 and | ||||
10 | // ULEB128 values. | ||||
11 | // | ||||
12 | //===----------------------------------------------------------------------===// | ||||
13 | |||||
14 | #ifndef LLVM_SUPPORT_LEB128_H | ||||
15 | #define LLVM_SUPPORT_LEB128_H | ||||
16 | |||||
17 | #include "llvm/Support/raw_ostream.h" | ||||
18 | |||||
19 | namespace llvm { | ||||
20 | |||||
21 | /// Utility function to encode a SLEB128 value to an output stream. Returns | ||||
22 | /// the length in bytes of the encoded value. | ||||
23 | inline unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, | ||||
24 | unsigned PadTo = 0) { | ||||
25 | bool More; | ||||
26 | unsigned Count = 0; | ||||
27 | do { | ||||
28 | uint8_t Byte = Value & 0x7f; | ||||
29 | // NOTE: this assumes that this signed shift is an arithmetic right shift. | ||||
30 | Value >>= 7; | ||||
31 | More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || | ||||
32 | ((Value == -1) && ((Byte & 0x40) != 0)))); | ||||
33 | Count++; | ||||
34 | if (More || Count < PadTo) | ||||
35 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
36 | OS << char(Byte); | ||||
37 | } while (More); | ||||
38 | |||||
39 | // Pad with 0x80 and emit a terminating byte at the end. | ||||
40 | if (Count < PadTo) { | ||||
41 | uint8_t PadValue = Value < 0 ? 0x7f : 0x00; | ||||
42 | for (; Count < PadTo - 1; ++Count) | ||||
43 | OS << char(PadValue | 0x80); | ||||
44 | OS << char(PadValue); | ||||
45 | Count++; | ||||
46 | } | ||||
47 | return Count; | ||||
48 | } | ||||
49 | |||||
50 | /// Utility function to encode a SLEB128 value to a buffer. Returns | ||||
51 | /// the length in bytes of the encoded value. | ||||
52 | inline unsigned encodeSLEB128(int64_t Value, uint8_t *p, unsigned PadTo = 0) { | ||||
53 | uint8_t *orig_p = p; | ||||
54 | unsigned Count = 0; | ||||
55 | bool More; | ||||
56 | do { | ||||
57 | uint8_t Byte = Value & 0x7f; | ||||
58 | // NOTE: this assumes that this signed shift is an arithmetic right shift. | ||||
59 | Value >>= 7; | ||||
60 | More = !((((Value == 0 ) && ((Byte & 0x40) == 0)) || | ||||
61 | ((Value == -1) && ((Byte & 0x40) != 0)))); | ||||
62 | Count++; | ||||
63 | if (More || Count < PadTo) | ||||
64 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
65 | *p++ = Byte; | ||||
66 | } while (More); | ||||
67 | |||||
68 | // Pad with 0x80 and emit a terminating byte at the end. | ||||
69 | if (Count < PadTo) { | ||||
70 | uint8_t PadValue = Value < 0 ? 0x7f : 0x00; | ||||
71 | for (; Count < PadTo - 1; ++Count) | ||||
72 | *p++ = (PadValue | 0x80); | ||||
73 | *p++ = PadValue; | ||||
74 | } | ||||
75 | return (unsigned)(p - orig_p); | ||||
76 | } | ||||
77 | |||||
78 | /// Utility function to encode a ULEB128 value to an output stream. Returns | ||||
79 | /// the length in bytes of the encoded value. | ||||
80 | inline unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, | ||||
81 | unsigned PadTo = 0) { | ||||
82 | unsigned Count = 0; | ||||
83 | do { | ||||
84 | uint8_t Byte = Value & 0x7f; | ||||
85 | Value >>= 7; | ||||
86 | Count++; | ||||
87 | if (Value != 0 || Count < PadTo) | ||||
88 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
89 | OS << char(Byte); | ||||
90 | } while (Value != 0); | ||||
91 | |||||
92 | // Pad with 0x80 and emit a null byte at the end. | ||||
93 | if (Count < PadTo) { | ||||
94 | for (; Count < PadTo - 1; ++Count) | ||||
95 | OS << '\x80'; | ||||
96 | OS << '\x00'; | ||||
97 | Count++; | ||||
98 | } | ||||
99 | return Count; | ||||
100 | } | ||||
101 | |||||
102 | /// Utility function to encode a ULEB128 value to a buffer. Returns | ||||
103 | /// the length in bytes of the encoded value. | ||||
104 | inline unsigned encodeULEB128(uint64_t Value, uint8_t *p, | ||||
105 | unsigned PadTo = 0) { | ||||
106 | uint8_t *orig_p = p; | ||||
107 | unsigned Count = 0; | ||||
108 | do { | ||||
109 | uint8_t Byte = Value & 0x7f; | ||||
110 | Value >>= 7; | ||||
111 | Count++; | ||||
112 | if (Value != 0 || Count < PadTo) | ||||
113 | Byte |= 0x80; // Mark this byte to show that more bytes will follow. | ||||
114 | *p++ = Byte; | ||||
115 | } while (Value != 0); | ||||
116 | |||||
117 | // Pad with 0x80 and emit a null byte at the end. | ||||
118 | if (Count < PadTo) { | ||||
119 | for (; Count < PadTo - 1; ++Count) | ||||
120 | *p++ = '\x80'; | ||||
121 | *p++ = '\x00'; | ||||
122 | } | ||||
123 | |||||
124 | return (unsigned)(p - orig_p); | ||||
125 | } | ||||
126 | |||||
127 | /// Utility function to decode a ULEB128 value. | ||||
128 | inline uint64_t decodeULEB128(const uint8_t *p, unsigned *n = nullptr, | ||||
129 | const uint8_t *end = nullptr, | ||||
130 | const char **error = nullptr) { | ||||
131 | const uint8_t *orig_p = p; | ||||
132 | uint64_t Value = 0; | ||||
133 | unsigned Shift = 0; | ||||
134 | if (error
| ||||
135 | *error = nullptr; | ||||
136 | do { | ||||
137 | if (p == end) { | ||||
138 | if (error
| ||||
139 | *error = "malformed uleb128, extends past end"; | ||||
140 | if (n
| ||||
141 | *n = (unsigned)(p - orig_p); | ||||
142 | return 0; | ||||
143 | } | ||||
144 | uint64_t Slice = *p & 0x7f; | ||||
145 | if ((Shift >= 64 && Slice != 0) || Slice << Shift >> Shift != Slice) { | ||||
146 | if (error) | ||||
147 | *error = "uleb128 too big for uint64"; | ||||
148 | if (n) | ||||
149 | *n = (unsigned)(p - orig_p); | ||||
150 | return 0; | ||||
151 | } | ||||
152 | Value += Slice << Shift; | ||||
153 | Shift += 7; | ||||
154 | } while (*p++ >= 128); | ||||
155 | if (n) | ||||
156 | *n = (unsigned)(p - orig_p); | ||||
157 | return Value; | ||||
158 | } | ||||
159 | |||||
160 | /// Utility function to decode a SLEB128 value. | ||||
161 | inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr, | ||||
162 | const uint8_t *end = nullptr, | ||||
163 | const char **error = nullptr) { | ||||
164 | const uint8_t *orig_p = p; | ||||
165 | int64_t Value = 0; | ||||
166 | unsigned Shift = 0; | ||||
167 | uint8_t Byte; | ||||
168 | if (error) | ||||
169 | *error = nullptr; | ||||
170 | do { | ||||
171 | if (p == end) { | ||||
172 | if (error) | ||||
173 | *error = "malformed sleb128, extends past end"; | ||||
174 | if (n) | ||||
175 | *n = (unsigned)(p - orig_p); | ||||
176 | return 0; | ||||
177 | } | ||||
178 | Byte = *p; | ||||
179 | uint64_t Slice = Byte & 0x7f; | ||||
180 | if ((Shift >= 64 && Slice != (Value < 0 ? 0x7f : 0x00)) || | ||||
181 | (Shift == 63 && Slice != 0 && Slice != 0x7f)) { | ||||
182 | if (error) | ||||
183 | *error = "sleb128 too big for int64"; | ||||
184 | if (n) | ||||
185 | *n = (unsigned)(p - orig_p); | ||||
186 | return 0; | ||||
187 | } | ||||
188 | Value |= Slice << Shift; | ||||
189 | Shift += 7; | ||||
190 | ++p; | ||||
191 | } while (Byte >= 128); | ||||
192 | // Sign extend negative numbers if needed. | ||||
193 | if (Shift < 64 && (Byte & 0x40)) | ||||
194 | Value |= (-1ULL) << Shift; | ||||
195 | if (n) | ||||
196 | *n = (unsigned)(p - orig_p); | ||||
197 | return Value; | ||||
198 | } | ||||
199 | |||||
200 | /// Utility function to get the size of the ULEB128-encoded value. | ||||
201 | extern unsigned getULEB128Size(uint64_t Value); | ||||
202 | |||||
203 | /// Utility function to get the size of the SLEB128-encoded value. | ||||
204 | extern unsigned getSLEB128Size(int64_t Value); | ||||
205 | |||||
206 | } // namespace llvm | ||||
207 | |||||
208 | #endif // LLVM_SUPPORT_LEB128_H |