Line data Source code
1 : //===- InstrProf.cpp - Instrumented profiling format support --------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file contains support for clang's instrumentation based PGO and
11 : // coverage.
12 : //
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "llvm/ProfileData/InstrProf.h"
16 : #include "llvm/ADT/ArrayRef.h"
17 : #include "llvm/ADT/SmallString.h"
18 : #include "llvm/ADT/SmallVector.h"
19 : #include "llvm/ADT/StringExtras.h"
20 : #include "llvm/ADT/StringRef.h"
21 : #include "llvm/ADT/Triple.h"
22 : #include "llvm/IR/Constant.h"
23 : #include "llvm/IR/Constants.h"
24 : #include "llvm/IR/Function.h"
25 : #include "llvm/IR/GlobalValue.h"
26 : #include "llvm/IR/GlobalVariable.h"
27 : #include "llvm/IR/Instruction.h"
28 : #include "llvm/IR/LLVMContext.h"
29 : #include "llvm/IR/MDBuilder.h"
30 : #include "llvm/IR/Metadata.h"
31 : #include "llvm/IR/Module.h"
32 : #include "llvm/IR/Type.h"
33 : #include "llvm/Support/Casting.h"
34 : #include "llvm/Support/CommandLine.h"
35 : #include "llvm/Support/Compiler.h"
36 : #include "llvm/Support/Compression.h"
37 : #include "llvm/Support/Endian.h"
38 : #include "llvm/Support/Error.h"
39 : #include "llvm/Support/ErrorHandling.h"
40 : #include "llvm/Support/LEB128.h"
41 : #include "llvm/Support/ManagedStatic.h"
42 : #include "llvm/Support/MathExtras.h"
43 : #include "llvm/Support/Path.h"
44 : #include "llvm/Support/SwapByteOrder.h"
45 : #include <algorithm>
46 : #include <cassert>
47 : #include <cstddef>
48 : #include <cstdint>
49 : #include <cstring>
50 : #include <memory>
51 : #include <string>
52 : #include <system_error>
53 : #include <utility>
54 : #include <vector>
55 :
56 : using namespace llvm;
57 :
58 : static cl::opt<bool> StaticFuncFullModulePrefix(
59 : "static-func-full-module-prefix", cl::init(true), cl::Hidden,
60 : cl::desc("Use full module build paths in the profile counter names for "
61 : "static functions."));
62 :
63 : // This option is tailored to users that have different top-level directory in
64 : // profile-gen and profile-use compilation. Users need to specific the number
65 : // of levels to strip. A value larger than the number of directories in the
66 : // source file will strip all the directory names and only leave the basename.
67 : //
68 : // Note current ThinLTO module importing for the indirect-calls assumes
69 : // the source directory name not being stripped. A non-zero option value here
70 : // can potentially prevent some inter-module indirect-call-promotions.
71 : static cl::opt<unsigned> StaticFuncStripDirNamePrefix(
72 : "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
73 : cl::desc("Strip specified level of directory name from source path in "
74 : "the profile counter name for static functions."));
75 :
76 59 : static std::string getInstrProfErrString(instrprof_error Err) {
77 59 : switch (Err) {
78 : case instrprof_error::success:
79 0 : return "Success";
80 : case instrprof_error::eof:
81 0 : return "End of File";
82 : case instrprof_error::unrecognized_format:
83 1 : return "Unrecognized instrumentation profile encoding format";
84 : case instrprof_error::bad_magic:
85 0 : return "Invalid instrumentation profile data (bad magic)";
86 : case instrprof_error::bad_header:
87 2 : return "Invalid instrumentation profile data (file header is corrupt)";
88 : case instrprof_error::unsupported_version:
89 0 : return "Unsupported instrumentation profile format version";
90 : case instrprof_error::unsupported_hash_type:
91 0 : return "Unsupported instrumentation profile hash type";
92 : case instrprof_error::too_large:
93 0 : return "Too much profile data";
94 : case instrprof_error::truncated:
95 1 : return "Truncated profile data";
96 : case instrprof_error::malformed:
97 10 : return "Malformed instrumentation profile data";
98 : case instrprof_error::unknown_function:
99 34 : return "No profile data available for function";
100 : case instrprof_error::hash_mismatch:
101 6 : return "Function control flow change detected (hash mismatch)";
102 : case instrprof_error::count_mismatch:
103 4 : return "Function basic block count change detected (counter mismatch)";
104 : case instrprof_error::counter_overflow:
105 1 : return "Counter overflow";
106 : case instrprof_error::value_site_count_mismatch:
107 0 : return "Function value site count change detected (counter mismatch)";
108 : case instrprof_error::compress_failed:
109 0 : return "Failed to compress data (zlib)";
110 : case instrprof_error::uncompress_failed:
111 0 : return "Failed to uncompress data (zlib)";
112 : case instrprof_error::empty_raw_profile:
113 0 : return "Empty raw profile file";
114 : case instrprof_error::zlib_unavailable:
115 0 : return "Profile uses zlib compression but the profile reader was built without zlib support";
116 : }
117 0 : llvm_unreachable("A value of instrprof_error has no message.");
118 : }
119 :
120 : namespace {
121 :
122 : // FIXME: This class is only here to support the transition to llvm::Error. It
123 : // will be removed once this transition is complete. Clients should prefer to
124 : // deal with the Error value directly, rather than converting to error_code.
125 0 : class InstrProfErrorCategoryType : public std::error_category {
126 0 : const char *name() const noexcept override { return "llvm.instrprof"; }
127 :
128 0 : std::string message(int IE) const override {
129 0 : return getInstrProfErrString(static_cast<instrprof_error>(IE));
130 : }
131 : };
132 :
133 : } // end anonymous namespace
134 :
135 : static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
136 :
137 0 : const std::error_category &llvm::instrprof_category() {
138 0 : return *ErrorCategory;
139 : }
140 :
141 : namespace {
142 :
143 : const char *InstrProfSectNameCommon[] = {
144 : #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
145 : SectNameCommon,
146 : #include "llvm/ProfileData/InstrProfData.inc"
147 : };
148 :
149 : const char *InstrProfSectNameCoff[] = {
150 : #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
151 : SectNameCoff,
152 : #include "llvm/ProfileData/InstrProfData.inc"
153 : };
154 :
155 : const char *InstrProfSectNamePrefix[] = {
156 : #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
157 : Prefix,
158 : #include "llvm/ProfileData/InstrProfData.inc"
159 : };
160 :
161 : } // namespace
162 :
163 : namespace llvm {
164 :
165 4726 : std::string getInstrProfSectionName(InstrProfSectKind IPSK,
166 : Triple::ObjectFormatType OF,
167 : bool AddSegmentInfo) {
168 : std::string SectName;
169 :
170 4726 : if (OF == Triple::MachO && AddSegmentInfo)
171 196 : SectName = InstrProfSectNamePrefix[IPSK];
172 :
173 4726 : if (OF == Triple::COFF)
174 51 : SectName += InstrProfSectNameCoff[IPSK];
175 : else
176 4675 : SectName += InstrProfSectNameCommon[IPSK];
177 :
178 4726 : if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
179 : SectName += ",regular,live_support";
180 :
181 4726 : return SectName;
182 : }
183 :
184 0 : void SoftInstrProfErrors::addError(instrprof_error IE) {
185 0 : if (IE == instrprof_error::success)
186 : return;
187 :
188 0 : if (FirstError == instrprof_error::success)
189 0 : FirstError = IE;
190 :
191 0 : switch (IE) {
192 0 : case instrprof_error::hash_mismatch:
193 0 : ++NumHashMismatches;
194 0 : break;
195 0 : case instrprof_error::count_mismatch:
196 0 : ++NumCountMismatches;
197 0 : break;
198 0 : case instrprof_error::counter_overflow:
199 0 : ++NumCounterOverflows;
200 0 : break;
201 0 : case instrprof_error::value_site_count_mismatch:
202 0 : ++NumValueSiteCountMismatches;
203 0 : break;
204 0 : default:
205 0 : llvm_unreachable("Not a soft error");
206 : }
207 : }
208 :
209 59 : std::string InstrProfError::message() const {
210 59 : return getInstrProfErrString(Err);
211 : }
212 :
213 : char InstrProfError::ID = 0;
214 :
215 2132 : std::string getPGOFuncName(StringRef RawFuncName,
216 : GlobalValue::LinkageTypes Linkage,
217 : StringRef FileName,
218 : uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
219 2132 : return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
220 : }
221 :
222 : // Strip NumPrefix level of directory name from PathNameStr. If the number of
223 : // directory separators is less than NumPrefix, strip all the directories and
224 : // leave base file name only.
225 4 : static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
226 : uint32_t Count = NumPrefix;
227 : uint32_t Pos = 0, LastPos = 0;
228 194 : for (auto & CI : PathNameStr) {
229 192 : ++Pos;
230 192 : if (llvm::sys::path::is_separator(CI)) {
231 : LastPos = Pos;
232 14 : --Count;
233 : }
234 192 : if (Count == 0)
235 : break;
236 : }
237 4 : return PathNameStr.substr(LastPos);
238 : }
239 :
240 : // Return the PGOFuncName. This function has some special handling when called
241 : // in LTO optimization. The following only applies when calling in LTO passes
242 : // (when \c InLTO is true): LTO's internalization privatizes many global linkage
243 : // symbols. This happens after value profile annotation, but those internal
244 : // linkage functions should not have a source prefix.
245 : // Additionally, for ThinLTO mode, exported internal functions are promoted
246 : // and renamed. We need to ensure that the original internal PGO name is
247 : // used when computing the GUID that is compared against the profiled GUIDs.
248 : // To differentiate compiler generated internal symbols from original ones,
249 : // PGOFuncName meta data are created and attached to the original internal
250 : // symbols in the value profile annotation step
251 : // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
252 : // data, its original linkage must be non-internal.
253 1682 : std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
254 1682 : if (!InLTO) {
255 : StringRef FileName = (StaticFuncFullModulePrefix
256 976 : ? F.getParent()->getName()
257 490 : : sys::path::filename(F.getParent()->getName()));
258 490 : if (StaticFuncFullModulePrefix && StaticFuncStripDirNamePrefix != 0)
259 4 : FileName = stripDirPrefix(FileName, StaticFuncStripDirNamePrefix);
260 490 : return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
261 : }
262 :
263 : // In LTO mode (when InLTO is true), first check if there is a meta data.
264 1192 : if (MDNode *MD = getPGOFuncNameMetadata(F)) {
265 7 : StringRef S = cast<MDString>(MD->getOperand(0))->getString();
266 7 : return S.str();
267 : }
268 :
269 : // If there is no meta data, the function must be a global before the value
270 : // profile annotation pass. Its current linkage may be internal if it is
271 : // internalized in LTO mode.
272 1185 : return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
273 : }
274 :
275 495 : StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
276 495 : if (FileName.empty())
277 0 : return PGOFuncName;
278 : // Drop the file name including ':'. See also getPGOFuncName.
279 : if (PGOFuncName.startswith(FileName))
280 16 : PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
281 495 : return PGOFuncName;
282 : }
283 :
284 : // \p FuncName is the string used as profile lookup key for the function. A
285 : // symbol is created to hold the name. Return the legalized symbol name.
286 398 : std::string getPGOFuncNameVarName(StringRef FuncName,
287 : GlobalValue::LinkageTypes Linkage) {
288 : std::string VarName = getInstrProfNameVarPrefix();
289 : VarName += FuncName;
290 :
291 : if (!GlobalValue::isLocalLinkage(Linkage))
292 : return VarName;
293 :
294 : // Now fix up illegal chars in local VarName that may upset the assembler.
295 : const char *InvalidChars = "-:<>/\"'";
296 : size_t found = VarName.find_first_of(InvalidChars);
297 357 : while (found != std::string::npos) {
298 62 : VarName[found] = '_';
299 62 : found = VarName.find_first_of(InvalidChars, found + 1);
300 : }
301 : return VarName;
302 : }
303 :
304 398 : GlobalVariable *createPGOFuncNameVar(Module &M,
305 : GlobalValue::LinkageTypes Linkage,
306 : StringRef PGOFuncName) {
307 : // We generally want to match the function's linkage, but available_externally
308 : // and extern_weak both have the wrong semantics, and anything that doesn't
309 : // need to link across compilation units doesn't need to be visible at all.
310 398 : if (Linkage == GlobalValue::ExternalWeakLinkage)
311 : Linkage = GlobalValue::LinkOnceAnyLinkage;
312 398 : else if (Linkage == GlobalValue::AvailableExternallyLinkage)
313 : Linkage = GlobalValue::LinkOnceODRLinkage;
314 792 : else if (Linkage == GlobalValue::InternalLinkage ||
315 396 : Linkage == GlobalValue::ExternalLinkage)
316 : Linkage = GlobalValue::PrivateLinkage;
317 :
318 : auto *Value =
319 398 : ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
320 : auto FuncNameVar =
321 398 : new GlobalVariable(M, Value->getType(), true, Linkage, Value,
322 1012 : getPGOFuncNameVarName(PGOFuncName, Linkage));
323 :
324 : // Hide the symbol so that we correctly get a copy for each executable.
325 : if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
326 : FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
327 :
328 398 : return FuncNameVar;
329 : }
330 :
331 93 : GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
332 93 : return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
333 : }
334 :
335 874 : Error InstrProfSymtab::create(Module &M, bool InLTO) {
336 2374 : for (Function &F : M) {
337 : // Function may not have a name: like using asm("") to overwrite the name.
338 : // Ignore in this case.
339 1500 : if (!F.hasName())
340 : continue;
341 1500 : const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
342 1500 : if (Error E = addFuncName(PGOFuncName))
343 : return E;
344 1500 : MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
345 : // In ThinLTO, local function may have been promoted to global and have
346 : // suffix added to the function name. We need to add the stripped function
347 : // name to the symbol table so that we can find a match from profile.
348 1500 : if (InLTO) {
349 1192 : auto pos = PGOFuncName.find('.');
350 1192 : if (pos != std::string::npos) {
351 91 : const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
352 91 : if (Error E = addFuncName(OtherFuncName))
353 : return E;
354 91 : MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
355 : }
356 : }
357 : }
358 874 : Sorted = false;
359 874 : finalizeSymtab();
360 : return Error::success();
361 : }
362 :
363 28 : uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
364 28 : finalizeSymtab();
365 : auto Result =
366 : std::lower_bound(AddrToMD5Map.begin(), AddrToMD5Map.end(), Address,
367 : [](const std::pair<uint64_t, uint64_t> &LHS,
368 0 : uint64_t RHS) { return LHS.first < RHS; });
369 : // Raw function pointer collected by value profiler may be from
370 : // external functions that are not instrumented. They won't have
371 : // mapping data to be used by the deserializer. Force the value to
372 : // be 0 in this case.
373 28 : if (Result != AddrToMD5Map.end() && Result->first == Address)
374 22 : return (uint64_t)Result->second;
375 : return 0;
376 : }
377 :
378 155 : Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
379 : bool doCompression, std::string &Result) {
380 : assert(!NameStrs.empty() && "No name data to emit");
381 :
382 155 : uint8_t Header[16], *P = Header;
383 : std::string UncompressedNameStrings =
384 155 : join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
385 :
386 : assert(StringRef(UncompressedNameStrings)
387 : .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
388 : "PGO name is invalid (contains separator token)");
389 :
390 155 : unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
391 155 : P += EncLen;
392 :
393 : auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
394 : EncLen = encodeULEB128(CompressedLen, P);
395 : P += EncLen;
396 : char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
397 : unsigned HeaderLen = P - &Header[0];
398 : Result.append(HeaderStr, HeaderLen);
399 : Result += InputStr;
400 : return Error::success();
401 155 : };
402 :
403 155 : if (!doCompression) {
404 4 : return WriteStringToResult(0, UncompressedNameStrings);
405 : }
406 :
407 : SmallString<128> CompressedNameStrings;
408 : Error E = zlib::compress(StringRef(UncompressedNameStrings),
409 151 : CompressedNameStrings, zlib::BestSizeCompression);
410 151 : if (E) {
411 0 : consumeError(std::move(E));
412 : return make_error<InstrProfError>(instrprof_error::compress_failed);
413 : }
414 :
415 151 : return WriteStringToResult(CompressedNameStrings.size(),
416 : CompressedNameStrings);
417 : }
418 :
419 785 : StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
420 : auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
421 : StringRef NameStr =
422 785 : Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
423 785 : return NameStr;
424 : }
425 :
426 147 : Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
427 : std::string &Result, bool doCompression) {
428 147 : std::vector<std::string> NameStrs;
429 548 : for (auto *NameVar : NameVars) {
430 802 : NameStrs.push_back(getPGOFuncNameVarInitializer(NameVar));
431 : }
432 : return collectPGOFuncNameStrings(
433 147 : NameStrs, zlib::isAvailable() && doCompression, Result);
434 : }
435 :
436 97 : Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
437 : const uint8_t *P = reinterpret_cast<const uint8_t *>(NameStrings.data());
438 : const uint8_t *EndP = reinterpret_cast<const uint8_t *>(NameStrings.data() +
439 97 : NameStrings.size());
440 246 : while (P < EndP) {
441 : uint32_t N;
442 149 : uint64_t UncompressedSize = decodeULEB128(P, &N);
443 149 : P += N;
444 149 : uint64_t CompressedSize = decodeULEB128(P, &N);
445 149 : P += N;
446 : bool isCompressed = (CompressedSize != 0);
447 : SmallString<128> UncompressedNameStrings;
448 149 : StringRef NameStrings;
449 149 : if (isCompressed) {
450 8 : if (!llvm::zlib::isAvailable())
451 : return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
452 :
453 : StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
454 : CompressedSize);
455 8 : if (Error E =
456 : zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
457 8 : UncompressedSize)) {
458 0 : consumeError(std::move(E));
459 : return make_error<InstrProfError>(instrprof_error::uncompress_failed);
460 : }
461 8 : P += CompressedSize;
462 16 : NameStrings = StringRef(UncompressedNameStrings.data(),
463 : UncompressedNameStrings.size());
464 : } else {
465 141 : NameStrings =
466 : StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
467 141 : P += UncompressedSize;
468 : }
469 : // Now parse the name strings.
470 : SmallVector<StringRef, 0> Names;
471 149 : NameStrings.split(Names, getInstrProfNameSeparator());
472 659 : for (StringRef &Name : Names)
473 1020 : if (Error E = Symtab.addFuncName(Name))
474 : return E;
475 :
476 411 : while (P < EndP && *P == 0)
477 262 : P++;
478 : }
479 : return Error::success();
480 : }
481 :
482 16 : void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
483 : uint64_t Weight,
484 : function_ref<void(instrprof_error)> Warn) {
485 : this->sortByTargetValues();
486 : Input.sortByTargetValues();
487 16 : auto I = ValueData.begin();
488 : auto IE = ValueData.end();
489 544 : for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
490 : ++J) {
491 1040 : while (I != IE && I->Value < J->Value)
492 : ++I;
493 528 : if (I != IE && I->Value == J->Value) {
494 : bool Overflowed;
495 16 : I->Count = SaturatingMultiplyAdd(J->Count, Weight, I->Count, &Overflowed);
496 16 : if (Overflowed)
497 2 : Warn(instrprof_error::counter_overflow);
498 : ++I;
499 : continue;
500 : }
501 512 : ValueData.insert(I, *J);
502 : }
503 16 : }
504 :
505 8 : void InstrProfValueSiteRecord::scale(uint64_t Weight,
506 : function_ref<void(instrprof_error)> Warn) {
507 20 : for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
508 : bool Overflowed;
509 12 : I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed);
510 12 : if (Overflowed)
511 0 : Warn(instrprof_error::counter_overflow);
512 : }
513 8 : }
514 :
515 : // Merge Value Profile data from Src record to this record for ValueKind.
516 : // Scale merged value counts by \p Weight.
517 174 : void InstrProfRecord::mergeValueProfData(
518 : uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
519 : function_ref<void(instrprof_error)> Warn) {
520 : uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
521 : uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
522 174 : if (ThisNumValueSites != OtherNumValueSites) {
523 0 : Warn(instrprof_error::value_site_count_mismatch);
524 0 : return;
525 : }
526 174 : if (!ThisNumValueSites)
527 : return;
528 : std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
529 6 : getOrCreateValueSitesForKind(ValueKind);
530 : MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
531 : Src.getValueSitesForKind(ValueKind);
532 22 : for (uint32_t I = 0; I < ThisNumValueSites; I++)
533 32 : ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
534 : }
535 :
536 91 : void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
537 : function_ref<void(instrprof_error)> Warn) {
538 : // If the number of counters doesn't match we either have bad data
539 : // or a hash collision.
540 273 : if (Counts.size() != Other.Counts.size()) {
541 : Warn(instrprof_error::count_mismatch);
542 4 : return;
543 : }
544 :
545 479 : for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
546 : bool Overflowed;
547 392 : Counts[I] =
548 1176 : SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
549 392 : if (Overflowed)
550 : Warn(instrprof_error::counter_overflow);
551 : }
552 :
553 261 : for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
554 174 : mergeValueProfData(Kind, Other, Weight, Warn);
555 : }
556 :
557 20 : void InstrProfRecord::scaleValueProfData(
558 : uint32_t ValueKind, uint64_t Weight,
559 : function_ref<void(instrprof_error)> Warn) {
560 28 : for (auto &R : getValueSitesForKind(ValueKind))
561 8 : R.scale(Weight, Warn);
562 20 : }
563 :
564 10 : void InstrProfRecord::scale(uint64_t Weight,
565 : function_ref<void(instrprof_error)> Warn) {
566 89 : for (auto &Count : this->Counts) {
567 : bool Overflowed;
568 79 : Count = SaturatingMultiply(Count, Weight, &Overflowed);
569 79 : if (Overflowed)
570 0 : Warn(instrprof_error::counter_overflow);
571 : }
572 30 : for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
573 20 : scaleValueProfData(Kind, Weight, Warn);
574 10 : }
575 :
576 : // Map indirect call target name hash to name string.
577 1949 : uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
578 : InstrProfSymtab *SymTab) {
579 1949 : if (!SymTab)
580 : return Value;
581 :
582 28 : if (ValueKind == IPVK_IndirectCallTarget)
583 28 : return SymTab->getFunctionHashFromAddress(Value);
584 :
585 : return Value;
586 : }
587 :
588 190 : void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
589 : InstrProfValueData *VData, uint32_t N,
590 : InstrProfSymtab *ValueMap) {
591 2139 : for (uint32_t I = 0; I < N; I++) {
592 1949 : VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
593 : }
594 : std::vector<InstrProfValueSiteRecord> &ValueSites =
595 190 : getOrCreateValueSitesForKind(ValueKind);
596 190 : if (N == 0)
597 47 : ValueSites.emplace_back();
598 : else
599 143 : ValueSites.emplace_back(VData, VData + N);
600 190 : }
601 :
602 : #define INSTR_PROF_COMMON_API_IMPL
603 : #include "llvm/ProfileData/InstrProfData.inc"
604 :
605 : /*!
606 : * ValueProfRecordClosure Interface implementation for InstrProfRecord
607 : * class. These C wrappers are used as adaptors so that C++ code can be
608 : * invoked as callbacks.
609 : */
610 421 : uint32_t getNumValueKindsInstrProf(const void *Record) {
611 421 : return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
612 : }
613 :
614 2518 : uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
615 : return reinterpret_cast<const InstrProfRecord *>(Record)
616 2518 : ->getNumValueSites(VKind);
617 : }
618 :
619 42 : uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
620 : return reinterpret_cast<const InstrProfRecord *>(Record)
621 42 : ->getNumValueData(VKind);
622 : }
623 :
624 71 : uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
625 : uint32_t S) {
626 : return reinterpret_cast<const InstrProfRecord *>(R)
627 71 : ->getNumValueDataForSite(VK, S);
628 : }
629 :
630 71 : void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
631 : uint32_t K, uint32_t S) {
632 71 : reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
633 71 : }
634 :
635 421 : ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
636 : ValueProfData *VD =
637 421 : (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
638 421 : memset(VD, 0, TotalSizeInBytes);
639 421 : return VD;
640 : }
641 :
642 : static ValueProfRecordClosure InstrProfRecordClosure = {
643 : nullptr,
644 : getNumValueKindsInstrProf,
645 : getNumValueSitesInstrProf,
646 : getNumValueDataInstrProf,
647 : getNumValueDataForSiteInstrProf,
648 : nullptr,
649 : getValueForSiteInstrProf,
650 : allocValueProfDataInstrProf};
651 :
652 : // Wrapper implementation using the closure mechanism.
653 417 : uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
654 417 : auto Closure = InstrProfRecordClosure;
655 417 : Closure.Record = &Record;
656 417 : return getValueProfDataSize(&Closure);
657 : }
658 :
659 : // Wrapper implementation using the closure mechanism.
660 : std::unique_ptr<ValueProfData>
661 421 : ValueProfData::serializeFrom(const InstrProfRecord &Record) {
662 421 : InstrProfRecordClosure.Record = &Record;
663 :
664 : std::unique_ptr<ValueProfData> VPD(
665 421 : serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
666 421 : return VPD;
667 : }
668 :
669 27 : void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
670 : InstrProfSymtab *SymTab) {
671 27 : Record.reserveSites(Kind, NumValueSites);
672 :
673 : InstrProfValueData *ValueData = getValueProfRecordValueData(this);
674 102 : for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
675 75 : uint8_t ValueDataCount = this->SiteCountArray[VSite];
676 75 : Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
677 75 : ValueData += ValueDataCount;
678 : }
679 27 : }
680 :
681 : // For writing/serializing, Old is the host endianness, and New is
682 : // byte order intended on disk. For Reading/deserialization, Old
683 : // is the on-disk source endianness, and New is the host endianness.
684 4 : void ValueProfRecord::swapBytes(support::endianness Old,
685 : support::endianness New) {
686 : using namespace support;
687 :
688 4 : if (Old == New)
689 : return;
690 :
691 4 : if (getHostEndianness() != Old) {
692 : sys::swapByteOrder<uint32_t>(NumValueSites);
693 : sys::swapByteOrder<uint32_t>(Kind);
694 : }
695 : uint32_t ND = getValueProfRecordNumValueData(this);
696 : InstrProfValueData *VD = getValueProfRecordValueData(this);
697 :
698 : // No need to swap byte array: SiteCountArrray.
699 28 : for (uint32_t I = 0; I < ND; I++) {
700 24 : sys::swapByteOrder<uint64_t>(VD[I].Value);
701 : sys::swapByteOrder<uint64_t>(VD[I].Count);
702 : }
703 4 : if (getHostEndianness() == Old) {
704 : sys::swapByteOrder<uint32_t>(NumValueSites);
705 : sys::swapByteOrder<uint32_t>(Kind);
706 : }
707 : }
708 :
709 908 : void ValueProfData::deserializeTo(InstrProfRecord &Record,
710 : InstrProfSymtab *SymTab) {
711 908 : if (NumValueKinds == 0)
712 : return;
713 :
714 : ValueProfRecord *VR = getFirstValueProfRecord(this);
715 53 : for (uint32_t K = 0; K < NumValueKinds; K++) {
716 27 : VR->deserializeTo(Record, SymTab);
717 : VR = getValueProfRecordNext(VR);
718 : }
719 : }
720 :
721 : template <class T>
722 : static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
723 : using namespace support;
724 :
725 904 : if (Orig == little)
726 : return endian::readNext<T, little, unaligned>(D);
727 : else
728 : return endian::readNext<T, big, unaligned>(D);
729 : }
730 :
731 : static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
732 904 : return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
733 904 : ValueProfData());
734 : }
735 :
736 904 : Error ValueProfData::checkIntegrity() {
737 904 : if (NumValueKinds > IPVK_Last + 1)
738 : return make_error<InstrProfError>(instrprof_error::malformed);
739 : // Total size needs to be mulltiple of quadword size.
740 904 : if (TotalSize % sizeof(uint64_t))
741 : return make_error<InstrProfError>(instrprof_error::malformed);
742 :
743 : ValueProfRecord *VR = getFirstValueProfRecord(this);
744 927 : for (uint32_t K = 0; K < this->NumValueKinds; K++) {
745 23 : if (VR->Kind > IPVK_Last)
746 : return make_error<InstrProfError>(instrprof_error::malformed);
747 : VR = getValueProfRecordNext(VR);
748 23 : if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
749 : return make_error<InstrProfError>(instrprof_error::malformed);
750 : }
751 : return Error::success();
752 : }
753 :
754 : Expected<std::unique_ptr<ValueProfData>>
755 904 : ValueProfData::getValueProfData(const unsigned char *D,
756 : const unsigned char *const BufferEnd,
757 : support::endianness Endianness) {
758 : using namespace support;
759 :
760 904 : if (D + sizeof(ValueProfData) > BufferEnd)
761 : return make_error<InstrProfError>(instrprof_error::truncated);
762 :
763 : const unsigned char *Header = D;
764 : uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
765 904 : if (D + TotalSize > BufferEnd)
766 : return make_error<InstrProfError>(instrprof_error::too_large);
767 :
768 : std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
769 904 : memcpy(VPD.get(), D, TotalSize);
770 : // Byte swap.
771 904 : VPD->swapBytesToHost(Endianness);
772 :
773 904 : Error E = VPD->checkIntegrity();
774 904 : if (E)
775 : return std::move(E);
776 :
777 : return std::move(VPD);
778 : }
779 :
780 904 : void ValueProfData::swapBytesToHost(support::endianness Endianness) {
781 : using namespace support;
782 :
783 904 : if (Endianness == getHostEndianness())
784 : return;
785 :
786 : sys::swapByteOrder<uint32_t>(TotalSize);
787 : sys::swapByteOrder<uint32_t>(NumValueKinds);
788 :
789 : ValueProfRecord *VR = getFirstValueProfRecord(this);
790 4 : for (uint32_t K = 0; K < NumValueKinds; K++) {
791 2 : VR->swapBytes(Endianness, getHostEndianness());
792 : VR = getValueProfRecordNext(VR);
793 : }
794 : }
795 :
796 417 : void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
797 : using namespace support;
798 :
799 417 : if (Endianness == getHostEndianness())
800 : return;
801 :
802 : ValueProfRecord *VR = getFirstValueProfRecord(this);
803 10 : for (uint32_t K = 0; K < NumValueKinds; K++) {
804 : ValueProfRecord *NVR = getValueProfRecordNext(VR);
805 2 : VR->swapBytes(getHostEndianness(), Endianness);
806 : VR = NVR;
807 : }
808 : sys::swapByteOrder<uint32_t>(TotalSize);
809 : sys::swapByteOrder<uint32_t>(NumValueKinds);
810 : }
811 :
812 10 : void annotateValueSite(Module &M, Instruction &Inst,
813 : const InstrProfRecord &InstrProfR,
814 : InstrProfValueKind ValueKind, uint32_t SiteIdx,
815 : uint32_t MaxMDCount) {
816 : uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
817 10 : if (!NV)
818 0 : return;
819 :
820 10 : uint64_t Sum = 0;
821 : std::unique_ptr<InstrProfValueData[]> VD =
822 10 : InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
823 :
824 : ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
825 10 : annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
826 : }
827 :
828 39 : void annotateValueSite(Module &M, Instruction &Inst,
829 : ArrayRef<InstrProfValueData> VDs,
830 : uint64_t Sum, InstrProfValueKind ValueKind,
831 : uint32_t MaxMDCount) {
832 39 : LLVMContext &Ctx = M.getContext();
833 : MDBuilder MDHelper(Ctx);
834 : SmallVector<Metadata *, 3> Vals;
835 : // Tag
836 39 : Vals.push_back(MDHelper.createString("VP"));
837 : // Value Kind
838 39 : Vals.push_back(MDHelper.createConstant(
839 39 : ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
840 : // Total Count
841 39 : Vals.push_back(
842 39 : MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
843 :
844 : // Value Profile Data
845 : uint32_t MDCount = MaxMDCount;
846 125 : for (auto &VD : VDs) {
847 115 : Vals.push_back(MDHelper.createConstant(
848 115 : ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
849 115 : Vals.push_back(MDHelper.createConstant(
850 115 : ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
851 115 : if (--MDCount == 0)
852 : break;
853 : }
854 39 : Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
855 39 : }
856 :
857 922 : bool getValueProfDataFromInst(const Instruction &Inst,
858 : InstrProfValueKind ValueKind,
859 : uint32_t MaxNumValueData,
860 : InstrProfValueData ValueData[],
861 : uint32_t &ActualNumValueData, uint64_t &TotalC) {
862 : MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
863 874 : if (!MD)
864 853 : return false;
865 :
866 69 : unsigned NOps = MD->getNumOperands();
867 :
868 69 : if (NOps < 5)
869 : return false;
870 :
871 : // Operand 0 is a string tag "VP":
872 : MDString *Tag = cast<MDString>(MD->getOperand(0));
873 69 : if (!Tag)
874 : return false;
875 :
876 138 : if (!Tag->getString().equals("VP"))
877 0 : return false;
878 :
879 : // Now check kind:
880 : ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
881 : if (!KindInt)
882 : return false;
883 69 : if (KindInt->getZExtValue() != ValueKind)
884 : return false;
885 :
886 : // Get total count
887 : ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
888 : if (!TotalCInt)
889 : return false;
890 69 : TotalC = TotalCInt->getZExtValue();
891 :
892 69 : ActualNumValueData = 0;
893 :
894 223 : for (unsigned I = 3; I < NOps; I += 2) {
895 169 : if (ActualNumValueData >= MaxNumValueData)
896 : break;
897 : ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
898 : ConstantInt *Count =
899 154 : mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
900 154 : if (!Value || !Count)
901 : return false;
902 308 : ValueData[ActualNumValueData].Value = Value->getZExtValue();
903 154 : ValueData[ActualNumValueData].Count = Count->getZExtValue();
904 154 : ActualNumValueData++;
905 : }
906 : return true;
907 : }
908 :
909 1216 : MDNode *getPGOFuncNameMetadata(const Function &F) {
910 1216 : return F.getMetadata(getPGOFuncNameMetadataName());
911 : }
912 :
913 497 : void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
914 : // Only for internal linkage functions.
915 497 : if (PGOFuncName == F.getName())
916 : return;
917 : // Don't create duplicated meta-data.
918 24 : if (getPGOFuncNameMetadata(F))
919 : return;
920 24 : LLVMContext &C = F.getContext();
921 24 : MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
922 24 : F.setMetadata(getPGOFuncNameMetadataName(), N);
923 : }
924 :
925 465 : bool needsComdatForCounter(const Function &F, const Module &M) {
926 465 : if (F.hasComdat())
927 : return true;
928 :
929 744 : if (!Triple(M.getTargetTriple()).supportsCOMDAT())
930 : return false;
931 :
932 : // See createPGOFuncNameVar for more details. To avoid link errors, profile
933 : // counters for function with available_externally linkage needs to be changed
934 : // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
935 : // created. Without using comdat, duplicate entries won't be removed by the
936 : // linker leading to increased data segement size and raw profile size. Even
937 : // worse, since the referenced counter from profile per-function data object
938 : // will be resolved to the common strong definition, the profile counts for
939 : // available_externally functions will end up being duplicated in raw profile
940 : // data. This can result in distorted profile as the counts of those dups
941 : // will be accumulated by the profile merger.
942 : GlobalValue::LinkageTypes Linkage = F.getLinkage();
943 588 : if (Linkage != GlobalValue::ExternalWeakLinkage &&
944 294 : Linkage != GlobalValue::AvailableExternallyLinkage)
945 286 : return false;
946 :
947 : return true;
948 : }
949 :
950 : // Check if INSTR_PROF_RAW_VERSION_VAR is defined.
951 852 : bool isIRPGOFlagSet(const Module *M) {
952 : auto IRInstrVar =
953 852 : M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
954 852 : if (!IRInstrVar || IRInstrVar->isDeclaration() ||
955 : IRInstrVar->hasLocalLinkage())
956 795 : return false;
957 :
958 : // Check if the flag is set.
959 57 : if (!IRInstrVar->hasInitializer())
960 : return false;
961 :
962 : const Constant *InitVal = IRInstrVar->getInitializer();
963 57 : if (!InitVal)
964 : return false;
965 :
966 57 : return (dyn_cast<ConstantInt>(InitVal)->getZExtValue() &
967 57 : VARIANT_MASK_IR_PROF) != 0;
968 : }
969 :
970 : // Check if we can safely rename this Comdat function.
971 81 : bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
972 81 : if (F.getName().empty())
973 : return false;
974 81 : if (!needsComdatForCounter(F, *(F.getParent())))
975 : return false;
976 : // Unsafe to rename the address-taken function (which can be used in
977 : // function comparison).
978 33 : if (CheckAddressTaken && F.hasAddressTaken())
979 : return false;
980 : // Only safe to do if this function may be discarded if it is not used
981 : // in the compilation unit.
982 : if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
983 : return false;
984 :
985 : // For AvailableExternallyLinkage functions.
986 : if (!F.hasComdat()) {
987 : assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage);
988 : return true;
989 : }
990 : return true;
991 : }
992 :
993 : // Parse the value profile options.
994 43344 : void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
995 : int64_t &RangeLast) {
996 : static const int64_t DefaultMemOPSizeRangeStart = 0;
997 : static const int64_t DefaultMemOPSizeRangeLast = 8;
998 43344 : RangeStart = DefaultMemOPSizeRangeStart;
999 43344 : RangeLast = DefaultMemOPSizeRangeLast;
1000 :
1001 43344 : if (!MemOPSizeRange.empty()) {
1002 0 : auto Pos = MemOPSizeRange.find(':');
1003 0 : if (Pos != std::string::npos) {
1004 0 : if (Pos > 0)
1005 0 : MemOPSizeRange.substr(0, Pos).getAsInteger(10, RangeStart);
1006 0 : if (Pos < MemOPSizeRange.size() - 1)
1007 0 : MemOPSizeRange.substr(Pos + 1).getAsInteger(10, RangeLast);
1008 : } else
1009 0 : MemOPSizeRange.getAsInteger(10, RangeLast);
1010 : }
1011 : assert(RangeLast >= RangeStart);
1012 43344 : }
1013 :
1014 : } // end namespace llvm
|
