LLVM  16.0.0git
InstrProf.cpp
Go to the documentation of this file.
1 //===- InstrProf.cpp - Instrumented profiling format support --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains support for clang's instrumentation based PGO and
10 // coverage.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SmallString.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/Config/config.h"
22 #include "llvm/IR/Constant.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/GlobalValue.h"
26 #include "llvm/IR/GlobalVariable.h"
27 #include "llvm/IR/Instruction.h"
28 #include "llvm/IR/LLVMContext.h"
29 #include "llvm/IR/MDBuilder.h"
30 #include "llvm/IR/Metadata.h"
31 #include "llvm/IR/Module.h"
32 #include "llvm/IR/Type.h"
34 #include "llvm/Support/Casting.h"
36 #include "llvm/Support/Compiler.h"
38 #include "llvm/Support/Endian.h"
39 #include "llvm/Support/Error.h"
41 #include "llvm/Support/LEB128.h"
43 #include "llvm/Support/Path.h"
45 #include <algorithm>
46 #include <cassert>
47 #include <cstddef>
48 #include <cstdint>
49 #include <cstring>
50 #include <memory>
51 #include <string>
52 #include <system_error>
53 #include <type_traits>
54 #include <utility>
55 #include <vector>
56 
57 using namespace llvm;
58 
60  "static-func-full-module-prefix", cl::init(true), cl::Hidden,
61  cl::desc("Use full module build paths in the profile counter names for "
62  "static functions."));
63 
64 // This option is tailored to users that have different top-level directory in
65 // profile-gen and profile-use compilation. Users need to specific the number
66 // of levels to strip. A value larger than the number of directories in the
67 // source file will strip all the directory names and only leave the basename.
68 //
69 // Note current ThinLTO module importing for the indirect-calls assumes
70 // the source directory name not being stripped. A non-zero option value here
71 // can potentially prevent some inter-module indirect-call-promotions.
73  "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
74  cl::desc("Strip specified level of directory name from source path in "
75  "the profile counter name for static functions."));
76 
77 static std::string getInstrProfErrString(instrprof_error Err,
78  const std::string &ErrMsg = "") {
79  std::string Msg;
81 
82  switch (Err) {
84  OS << "success";
85  break;
87  OS << "end of File";
88  break;
90  OS << "unrecognized instrumentation profile encoding format";
91  break;
93  OS << "invalid instrumentation profile data (bad magic)";
94  break;
96  OS << "invalid instrumentation profile data (file header is corrupt)";
97  break;
99  OS << "unsupported instrumentation profile format version";
100  break;
102  OS << "unsupported instrumentation profile hash type";
103  break;
105  OS << "too much profile data";
106  break;
108  OS << "truncated profile data";
109  break;
111  OS << "malformed instrumentation profile data";
112  break;
114  OS << "debug info for correlation is required";
115  break;
117  OS << "debug info for correlation is not necessary";
118  break;
120  OS << "unable to correlate profile";
121  break;
123  OS << "invalid profile created. Please file a bug "
124  "at: " BUG_REPORT_URL
125  " and include the profraw files that caused this error.";
126  break;
128  OS << "no profile data available for function";
129  break;
131  OS << "function control flow change detected (hash mismatch)";
132  break;
134  OS << "function basic block count change detected (counter mismatch)";
135  break;
137  OS << "counter overflow";
138  break;
140  OS << "function value site count change detected (counter mismatch)";
141  break;
143  OS << "failed to compress data (zlib)";
144  break;
146  OS << "failed to uncompress data (zlib)";
147  break;
149  OS << "empty raw profile file";
150  break;
152  OS << "profile uses zlib compression but the profile reader was built "
153  "without zlib support";
154  break;
155  }
156 
157  // If optional error message is not empty, append it to the message.
158  if (!ErrMsg.empty())
159  OS << ": " << ErrMsg;
160 
161  return OS.str();
162 }
163 
164 namespace {
165 
166 // FIXME: This class is only here to support the transition to llvm::Error. It
167 // will be removed once this transition is complete. Clients should prefer to
168 // deal with the Error value directly, rather than converting to error_code.
169 class InstrProfErrorCategoryType : public std::error_category {
170  const char *name() const noexcept override { return "llvm.instrprof"; }
171 
172  std::string message(int IE) const override {
173  return getInstrProfErrString(static_cast<instrprof_error>(IE));
174  }
175 };
176 
177 } // end anonymous namespace
178 
179 const std::error_category &llvm::instrprof_category() {
180  static InstrProfErrorCategoryType ErrorCategory;
181  return ErrorCategory;
182 }
183 
184 namespace {
185 
186 const char *InstrProfSectNameCommon[] = {
187 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
188  SectNameCommon,
190 };
191 
192 const char *InstrProfSectNameCoff[] = {
193 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
194  SectNameCoff,
196 };
197 
198 const char *InstrProfSectNamePrefix[] = {
199 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
200  Prefix,
202 };
203 
204 } // namespace
205 
206 namespace llvm {
207 
209  "enable-name-compression",
210  cl::desc("Enable name/filename string compression"), cl::init(true));
211 
214  bool AddSegmentInfo) {
215  std::string SectName;
216 
217  if (OF == Triple::MachO && AddSegmentInfo)
218  SectName = InstrProfSectNamePrefix[IPSK];
219 
220  if (OF == Triple::COFF)
221  SectName += InstrProfSectNameCoff[IPSK];
222  else
223  SectName += InstrProfSectNameCommon[IPSK];
224 
225  if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
226  SectName += ",regular,live_support";
227 
228  return SectName;
229 }
230 
233  return;
234 
235  if (FirstError == instrprof_error::success)
236  FirstError = IE;
237 
238  switch (IE) {
240  ++NumHashMismatches;
241  break;
243  ++NumCountMismatches;
244  break;
246  ++NumCounterOverflows;
247  break;
249  ++NumValueSiteCountMismatches;
250  break;
251  default:
252  llvm_unreachable("Not a soft error");
253  }
254 }
255 
256 std::string InstrProfError::message() const {
257  return getInstrProfErrString(Err, Msg);
258 }
259 
260 char InstrProfError::ID = 0;
261 
262 std::string getPGOFuncName(StringRef RawFuncName,
264  StringRef FileName,
266  return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
267 }
268 
269 // Strip NumPrefix level of directory name from PathNameStr. If the number of
270 // directory separators is less than NumPrefix, strip all the directories and
271 // leave base file name only.
272 static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
273  uint32_t Count = NumPrefix;
274  uint32_t Pos = 0, LastPos = 0;
275  for (auto & CI : PathNameStr) {
276  ++Pos;
278  LastPos = Pos;
279  --Count;
280  }
281  if (Count == 0)
282  break;
283  }
284  return PathNameStr.substr(LastPos);
285 }
286 
287 // Return the PGOFuncName. This function has some special handling when called
288 // in LTO optimization. The following only applies when calling in LTO passes
289 // (when \c InLTO is true): LTO's internalization privatizes many global linkage
290 // symbols. This happens after value profile annotation, but those internal
291 // linkage functions should not have a source prefix.
292 // Additionally, for ThinLTO mode, exported internal functions are promoted
293 // and renamed. We need to ensure that the original internal PGO name is
294 // used when computing the GUID that is compared against the profiled GUIDs.
295 // To differentiate compiler generated internal symbols from original ones,
296 // PGOFuncName meta data are created and attached to the original internal
297 // symbols in the value profile annotation step
298 // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
299 // data, its original linkage must be non-internal.
300 std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
301  if (!InLTO) {
302  StringRef FileName(F.getParent()->getSourceFileName());
303  uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
304  if (StripLevel < StaticFuncStripDirNamePrefix)
305  StripLevel = StaticFuncStripDirNamePrefix;
306  if (StripLevel)
307  FileName = stripDirPrefix(FileName, StripLevel);
308  return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
309  }
310 
311  // In LTO mode (when InLTO is true), first check if there is a meta data.
312  if (MDNode *MD = getPGOFuncNameMetadata(F)) {
313  StringRef S = cast<MDString>(MD->getOperand(0))->getString();
314  return S.str();
315  }
316 
317  // If there is no meta data, the function must be a global before the value
318  // profile annotation pass. Its current linkage may be internal if it is
319  // internalized in LTO mode.
320  return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
321 }
322 
324  if (FileName.empty())
325  return PGOFuncName;
326  // Drop the file name including ':'. See also getPGOFuncName.
327  if (PGOFuncName.startswith(FileName))
328  PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
329  return PGOFuncName;
330 }
331 
332 // \p FuncName is the string used as profile lookup key for the function. A
333 // symbol is created to hold the name. Return the legalized symbol name.
334 std::string getPGOFuncNameVarName(StringRef FuncName,
335  GlobalValue::LinkageTypes Linkage) {
336  std::string VarName = std::string(getInstrProfNameVarPrefix());
337  VarName += FuncName;
338 
339  if (!GlobalValue::isLocalLinkage(Linkage))
340  return VarName;
341 
342  // Now fix up illegal chars in local VarName that may upset the assembler.
343  const char *InvalidChars = "-:<>/\"'";
344  size_t found = VarName.find_first_of(InvalidChars);
345  while (found != std::string::npos) {
346  VarName[found] = '_';
347  found = VarName.find_first_of(InvalidChars, found + 1);
348  }
349  return VarName;
350 }
351 
354  StringRef PGOFuncName) {
355  // We generally want to match the function's linkage, but available_externally
356  // and extern_weak both have the wrong semantics, and anything that doesn't
357  // need to link across compilation units doesn't need to be visible at all.
358  if (Linkage == GlobalValue::ExternalWeakLinkage)
360  else if (Linkage == GlobalValue::AvailableExternallyLinkage)
362  else if (Linkage == GlobalValue::InternalLinkage ||
363  Linkage == GlobalValue::ExternalLinkage)
364  Linkage = GlobalValue::PrivateLinkage;
365 
366  auto *Value =
367  ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
368  auto FuncNameVar =
369  new GlobalVariable(M, Value->getType(), true, Linkage, Value,
370  getPGOFuncNameVarName(PGOFuncName, Linkage));
371 
372  // Hide the symbol so that we correctly get a copy for each executable.
373  if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
374  FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
375 
376  return FuncNameVar;
377 }
378 
380  return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
381 }
382 
384  for (Function &F : M) {
385  // Function may not have a name: like using asm("") to overwrite the name.
386  // Ignore in this case.
387  if (!F.hasName())
388  continue;
389  const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
390  if (Error E = addFuncName(PGOFuncName))
391  return E;
392  MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
393  // In ThinLTO, local function may have been promoted to global and have
394  // suffix ".llvm." added to the function name. We need to add the
395  // stripped function name to the symbol table so that we can find a match
396  // from profile.
397  //
398  // We may have other suffixes similar as ".llvm." which are needed to
399  // be stripped before the matching, but ".__uniq." suffix which is used
400  // to differentiate internal linkage functions in different modules
401  // should be kept. Now this is the only suffix with the pattern ".xxx"
402  // which is kept before matching.
403  const std::string UniqSuffix = ".__uniq.";
404  auto pos = PGOFuncName.find(UniqSuffix);
405  // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
406  // search '.' from the beginning.
407  if (pos != std::string::npos)
408  pos += UniqSuffix.length();
409  else
410  pos = 0;
411  pos = PGOFuncName.find('.', pos);
412  if (pos != std::string::npos && pos != 0) {
413  const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
414  if (Error E = addFuncName(OtherFuncName))
415  return E;
416  MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
417  }
418  }
419  Sorted = false;
420  finalizeSymtab();
421  return Error::success();
422 }
423 
425  finalizeSymtab();
426  auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
427  return A.first < Address;
428  });
429  // Raw function pointer collected by value profiler may be from
430  // external functions that are not instrumented. They won't have
431  // mapping data to be used by the deserializer. Force the value to
432  // be 0 in this case.
433  if (It != AddrToMD5Map.end() && It->first == Address)
434  return (uint64_t)It->second;
435  return 0;
436 }
437 
439  bool doCompression, std::string &Result) {
440  assert(!NameStrs.empty() && "No name data to emit");
441 
442  uint8_t Header[16], *P = Header;
443  std::string UncompressedNameStrings =
444  join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
445 
446  assert(StringRef(UncompressedNameStrings)
447  .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
448  "PGO name is invalid (contains separator token)");
449 
450  unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
451  P += EncLen;
452 
453  auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
454  EncLen = encodeULEB128(CompressedLen, P);
455  P += EncLen;
456  char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
457  unsigned HeaderLen = P - &Header[0];
458  Result.append(HeaderStr, HeaderLen);
459  Result += InputStr;
460  return Error::success();
461  };
462 
463  if (!doCompression) {
464  return WriteStringToResult(0, UncompressedNameStrings);
465  }
466 
467  SmallVector<uint8_t, 128> CompressedNameStrings;
468  compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings),
469  CompressedNameStrings,
471 
472  return WriteStringToResult(CompressedNameStrings.size(),
473  toStringRef(CompressedNameStrings));
474 }
475 
477  auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
478  StringRef NameStr =
479  Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
480  return NameStr;
481 }
482 
484  std::string &Result, bool doCompression) {
485  std::vector<std::string> NameStrs;
486  for (auto *NameVar : NameVars) {
487  NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
488  }
490  NameStrs, compression::zlib::isAvailable() && doCompression, Result);
491 }
492 
494  const uint8_t *P = NameStrings.bytes_begin();
495  const uint8_t *EndP = NameStrings.bytes_end();
496  while (P < EndP) {
497  uint32_t N;
498  uint64_t UncompressedSize = decodeULEB128(P, &N);
499  P += N;
500  uint64_t CompressedSize = decodeULEB128(P, &N);
501  P += N;
502  bool isCompressed = (CompressedSize != 0);
503  SmallVector<uint8_t, 128> UncompressedNameStrings;
504  StringRef NameStrings;
505  if (isCompressed) {
507  return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
508 
510  makeArrayRef(P, CompressedSize), UncompressedNameStrings,
511  UncompressedSize)) {
513  return make_error<InstrProfError>(instrprof_error::uncompress_failed);
514  }
515  P += CompressedSize;
516  NameStrings = toStringRef(UncompressedNameStrings);
517  } else {
518  NameStrings =
519  StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
520  P += UncompressedSize;
521  }
522  // Now parse the name strings.
524  NameStrings.split(Names, getInstrProfNameSeparator());
525  for (StringRef &Name : Names)
526  if (Error E = Symtab.addFuncName(Name))
527  return E;
528 
529  while (P < EndP && *P == 0)
530  P++;
531  }
532  return Error::success();
533 }
534 
536  uint64_t FuncSum = 0;
537  Sum.NumEntries += Counts.size();
538  for (uint64_t Count : Counts)
539  FuncSum += Count;
540  Sum.CountSum += FuncSum;
541 
542  for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
543  uint64_t KindSum = 0;
544  uint32_t NumValueSites = getNumValueSites(VK);
545  for (size_t I = 0; I < NumValueSites; ++I) {
547  std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I);
548  for (uint32_t V = 0; V < NV; V++)
549  KindSum += VD[V].Count;
550  }
551  Sum.ValueCounts[VK] += KindSum;
552  }
553 }
554 
557  OverlapStats &Overlap,
558  OverlapStats &FuncLevelOverlap) {
559  this->sortByTargetValues();
560  Input.sortByTargetValues();
561  double Score = 0.0f, FuncLevelScore = 0.0f;
562  auto I = ValueData.begin();
563  auto IE = ValueData.end();
564  auto J = Input.ValueData.begin();
565  auto JE = Input.ValueData.end();
566  while (I != IE && J != JE) {
567  if (I->Value == J->Value) {
568  Score += OverlapStats::score(I->Count, J->Count,
569  Overlap.Base.ValueCounts[ValueKind],
570  Overlap.Test.ValueCounts[ValueKind]);
571  FuncLevelScore += OverlapStats::score(
572  I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
573  FuncLevelOverlap.Test.ValueCounts[ValueKind]);
574  ++I;
575  } else if (I->Value < J->Value) {
576  ++I;
577  continue;
578  }
579  ++J;
580  }
581  Overlap.Overlap.ValueCounts[ValueKind] += Score;
582  FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
583 }
584 
585 // Return false on mismatch.
587  InstrProfRecord &Other,
588  OverlapStats &Overlap,
589  OverlapStats &FuncLevelOverlap) {
590  uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
591  assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));
592  if (!ThisNumValueSites)
593  return;
594 
595  std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
596  getOrCreateValueSitesForKind(ValueKind);
598  Other.getValueSitesForKind(ValueKind);
599  for (uint32_t I = 0; I < ThisNumValueSites; I++)
600  ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
601  FuncLevelOverlap);
602 }
603 
605  OverlapStats &FuncLevelOverlap,
606  uint64_t ValueCutoff) {
607  // FuncLevel CountSum for other should already computed and nonzero.
608  assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
609  accumulateCounts(FuncLevelOverlap.Base);
610  bool Mismatch = (Counts.size() != Other.Counts.size());
611 
612  // Check if the value profiles mismatch.
613  if (!Mismatch) {
614  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
615  uint32_t ThisNumValueSites = getNumValueSites(Kind);
616  uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
617  if (ThisNumValueSites != OtherNumValueSites) {
618  Mismatch = true;
619  break;
620  }
621  }
622  }
623  if (Mismatch) {
624  Overlap.addOneMismatch(FuncLevelOverlap.Test);
625  return;
626  }
627 
628  // Compute overlap for value counts.
629  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
630  overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
631 
632  double Score = 0.0;
633  uint64_t MaxCount = 0;
634  // Compute overlap for edge counts.
635  for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
636  Score += OverlapStats::score(Counts[I], Other.Counts[I],
637  Overlap.Base.CountSum, Overlap.Test.CountSum);
638  MaxCount = std::max(Other.Counts[I], MaxCount);
639  }
640  Overlap.Overlap.CountSum += Score;
641  Overlap.Overlap.NumEntries += 1;
642 
643  if (MaxCount >= ValueCutoff) {
644  double FuncScore = 0.0;
645  for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
646  FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
647  FuncLevelOverlap.Base.CountSum,
648  FuncLevelOverlap.Test.CountSum);
649  FuncLevelOverlap.Overlap.CountSum = FuncScore;
650  FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
651  FuncLevelOverlap.Valid = true;
652  }
653 }
654 
656  uint64_t Weight,
657  function_ref<void(instrprof_error)> Warn) {
658  this->sortByTargetValues();
659  Input.sortByTargetValues();
660  auto I = ValueData.begin();
661  auto IE = ValueData.end();
662  for (const InstrProfValueData &J : Input.ValueData) {
663  while (I != IE && I->Value < J.Value)
664  ++I;
665  if (I != IE && I->Value == J.Value) {
666  bool Overflowed;
667  I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed);
668  if (Overflowed)
670  ++I;
671  continue;
672  }
673  ValueData.insert(I, J);
674  }
675 }
676 
678  function_ref<void(instrprof_error)> Warn) {
679  for (InstrProfValueData &I : ValueData) {
680  bool Overflowed;
681  I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
682  if (Overflowed)
684  }
685 }
686 
687 // Merge Value Profile data from Src record to this record for ValueKind.
688 // Scale merged value counts by \p Weight.
689 void InstrProfRecord::mergeValueProfData(
691  function_ref<void(instrprof_error)> Warn) {
692  uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
693  uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
694  if (ThisNumValueSites != OtherNumValueSites) {
696  return;
697  }
698  if (!ThisNumValueSites)
699  return;
700  std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
701  getOrCreateValueSitesForKind(ValueKind);
703  Src.getValueSitesForKind(ValueKind);
704  for (uint32_t I = 0; I < ThisNumValueSites; I++)
705  ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
706 }
707 
709  function_ref<void(instrprof_error)> Warn) {
710  // If the number of counters doesn't match we either have bad data
711  // or a hash collision.
712  if (Counts.size() != Other.Counts.size()) {
714  return;
715  }
716 
717  // Special handling of the first count as the PseudoCount.
718  CountPseudoKind OtherKind = Other.getCountPseudoKind();
719  CountPseudoKind ThisKind = getCountPseudoKind();
720  if (OtherKind != NotPseudo || ThisKind != NotPseudo) {
721  // We don't allow the merge of a profile with pseudo counts and
722  // a normal profile (i.e. without pesudo counts).
723  // Profile supplimenation should be done after the profile merge.
724  if (OtherKind == NotPseudo || ThisKind == NotPseudo) {
726  return;
727  }
728  if (OtherKind == PseudoHot || ThisKind == PseudoHot)
730  else
732  return;
733  }
734 
735  for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
736  bool Overflowed;
737  uint64_t Value =
738  SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
739  if (Value > getInstrMaxCountValue()) {
741  Overflowed = true;
742  }
743  Counts[I] = Value;
744  if (Overflowed)
746  }
747 
748  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
749  mergeValueProfData(Kind, Other, Weight, Warn);
750 }
751 
752 void InstrProfRecord::scaleValueProfData(
754  function_ref<void(instrprof_error)> Warn) {
755  for (auto &R : getValueSitesForKind(ValueKind))
756  R.scale(N, D, Warn);
757 }
758 
760  function_ref<void(instrprof_error)> Warn) {
761  assert(D != 0 && "D cannot be 0");
762  for (auto &Count : this->Counts) {
763  bool Overflowed;
764  Count = SaturatingMultiply(Count, N, &Overflowed) / D;
765  if (Count > getInstrMaxCountValue()) {
766  Count = getInstrMaxCountValue();
767  Overflowed = true;
768  }
769  if (Overflowed)
771  }
772  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
773  scaleValueProfData(Kind, N, D, Warn);
774 }
775 
776 // Map indirect call target name hash to name string.
777 uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
778  InstrProfSymtab *SymTab) {
779  if (!SymTab)
780  return Value;
781 
782  if (ValueKind == IPVK_IndirectCallTarget)
783  return SymTab->getFunctionHashFromAddress(Value);
784 
785  return Value;
786 }
787 
789  InstrProfValueData *VData, uint32_t N,
791  for (uint32_t I = 0; I < N; I++) {
792  VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
793  }
794  std::vector<InstrProfValueSiteRecord> &ValueSites =
795  getOrCreateValueSitesForKind(ValueKind);
796  if (N == 0)
797  ValueSites.emplace_back();
798  else
799  ValueSites.emplace_back(VData, VData + N);
800 }
801 
802 #define INSTR_PROF_COMMON_API_IMPL
804 
805 /*!
806  * ValueProfRecordClosure Interface implementation for InstrProfRecord
807  * class. These C wrappers are used as adaptors so that C++ code can be
808  * invoked as callbacks.
809  */
811  return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
812 }
813 
815  return reinterpret_cast<const InstrProfRecord *>(Record)
816  ->getNumValueSites(VKind);
817 }
818 
820  return reinterpret_cast<const InstrProfRecord *>(Record)
821  ->getNumValueData(VKind);
822 }
823 
825  uint32_t S) {
826  return reinterpret_cast<const InstrProfRecord *>(R)
827  ->getNumValueDataForSite(VK, S);
828 }
829 
830 void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
831  uint32_t K, uint32_t S) {
832  reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
833 }
834 
835 ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
836  ValueProfData *VD =
837  (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
838  memset(VD, 0, TotalSizeInBytes);
839  return VD;
840 }
841 
842 static ValueProfRecordClosure InstrProfRecordClosure = {
843  nullptr,
848  nullptr,
851 
852 // Wrapper implementation using the closure mechanism.
853 uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
854  auto Closure = InstrProfRecordClosure;
855  Closure.Record = &Record;
856  return getValueProfDataSize(&Closure);
857 }
858 
859 // Wrapper implementation using the closure mechanism.
860 std::unique_ptr<ValueProfData>
861 ValueProfData::serializeFrom(const InstrProfRecord &Record) {
862  InstrProfRecordClosure.Record = &Record;
863 
864  std::unique_ptr<ValueProfData> VPD(
865  serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
866  return VPD;
867 }
868 
869 void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
870  InstrProfSymtab *SymTab) {
871  Record.reserveSites(Kind, NumValueSites);
872 
873  InstrProfValueData *ValueData = getValueProfRecordValueData(this);
874  for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
875  uint8_t ValueDataCount = this->SiteCountArray[VSite];
876  Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
877  ValueData += ValueDataCount;
878  }
879 }
880 
881 // For writing/serializing, Old is the host endianness, and New is
882 // byte order intended on disk. For Reading/deserialization, Old
883 // is the on-disk source endianness, and New is the host endianness.
884 void ValueProfRecord::swapBytes(support::endianness Old,
885  support::endianness New) {
886  using namespace support;
887 
888  if (Old == New)
889  return;
890 
891  if (getHostEndianness() != Old) {
892  sys::swapByteOrder<uint32_t>(NumValueSites);
893  sys::swapByteOrder<uint32_t>(Kind);
894  }
895  uint32_t ND = getValueProfRecordNumValueData(this);
896  InstrProfValueData *VD = getValueProfRecordValueData(this);
897 
898  // No need to swap byte array: SiteCountArrray.
899  for (uint32_t I = 0; I < ND; I++) {
900  sys::swapByteOrder<uint64_t>(VD[I].Value);
901  sys::swapByteOrder<uint64_t>(VD[I].Count);
902  }
903  if (getHostEndianness() == Old) {
904  sys::swapByteOrder<uint32_t>(NumValueSites);
905  sys::swapByteOrder<uint32_t>(Kind);
906  }
907 }
908 
909 void ValueProfData::deserializeTo(InstrProfRecord &Record,
910  InstrProfSymtab *SymTab) {
911  if (NumValueKinds == 0)
912  return;
913 
914  ValueProfRecord *VR = getFirstValueProfRecord(this);
915  for (uint32_t K = 0; K < NumValueKinds; K++) {
916  VR->deserializeTo(Record, SymTab);
917  VR = getValueProfRecordNext(VR);
918  }
919 }
920 
921 template <class T>
922 static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
923  using namespace support;
924 
925  if (Orig == little)
926  return endian::readNext<T, little, unaligned>(D);
927  else
928  return endian::readNext<T, big, unaligned>(D);
929 }
930 
931 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
932  return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
933  ValueProfData());
934 }
935 
936 Error ValueProfData::checkIntegrity() {
937  if (NumValueKinds > IPVK_Last + 1)
938  return make_error<InstrProfError>(
939  instrprof_error::malformed, "number of value profile kinds is invalid");
940  // Total size needs to be multiple of quadword size.
941  if (TotalSize % sizeof(uint64_t))
942  return make_error<InstrProfError>(
943  instrprof_error::malformed, "total size is not multiples of quardword");
944 
945  ValueProfRecord *VR = getFirstValueProfRecord(this);
946  for (uint32_t K = 0; K < this->NumValueKinds; K++) {
947  if (VR->Kind > IPVK_Last)
948  return make_error<InstrProfError>(instrprof_error::malformed,
949  "value kind is invalid");
950  VR = getValueProfRecordNext(VR);
951  if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
952  return make_error<InstrProfError>(
954  "value profile address is greater than total size");
955  }
956  return Error::success();
957 }
958 
960 ValueProfData::getValueProfData(const unsigned char *D,
961  const unsigned char *const BufferEnd,
963  using namespace support;
964 
965  if (D + sizeof(ValueProfData) > BufferEnd)
966  return make_error<InstrProfError>(instrprof_error::truncated);
967 
968  const unsigned char *Header = D;
969  uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
970  if (D + TotalSize > BufferEnd)
971  return make_error<InstrProfError>(instrprof_error::too_large);
972 
973  std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
974  memcpy(VPD.get(), D, TotalSize);
975  // Byte swap.
976  VPD->swapBytesToHost(Endianness);
977 
978  Error E = VPD->checkIntegrity();
979  if (E)
980  return std::move(E);
981 
982  return std::move(VPD);
983 }
984 
985 void ValueProfData::swapBytesToHost(support::endianness Endianness) {
986  using namespace support;
987 
988  if (Endianness == getHostEndianness())
989  return;
990 
991  sys::swapByteOrder<uint32_t>(TotalSize);
992  sys::swapByteOrder<uint32_t>(NumValueKinds);
993 
994  ValueProfRecord *VR = getFirstValueProfRecord(this);
995  for (uint32_t K = 0; K < NumValueKinds; K++) {
996  VR->swapBytes(Endianness, getHostEndianness());
997  VR = getValueProfRecordNext(VR);
998  }
999 }
1000 
1001 void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
1002  using namespace support;
1003 
1004  if (Endianness == getHostEndianness())
1005  return;
1006 
1007  ValueProfRecord *VR = getFirstValueProfRecord(this);
1008  for (uint32_t K = 0; K < NumValueKinds; K++) {
1009  ValueProfRecord *NVR = getValueProfRecordNext(VR);
1010  VR->swapBytes(getHostEndianness(), Endianness);
1011  VR = NVR;
1012  }
1013  sys::swapByteOrder<uint32_t>(TotalSize);
1014  sys::swapByteOrder<uint32_t>(NumValueKinds);
1015 }
1016 
1018  const InstrProfRecord &InstrProfR,
1020  uint32_t MaxMDCount) {
1021  uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
1022  if (!NV)
1023  return;
1024 
1025  uint64_t Sum = 0;
1026  std::unique_ptr<InstrProfValueData[]> VD =
1027  InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
1028 
1029  ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
1030  annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1031 }
1032 
1036  uint32_t MaxMDCount) {
1037  LLVMContext &Ctx = M.getContext();
1038  MDBuilder MDHelper(Ctx);
1040  // Tag
1041  Vals.push_back(MDHelper.createString("VP"));
1042  // Value Kind
1043  Vals.push_back(MDHelper.createConstant(
1045  // Total Count
1046  Vals.push_back(
1047  MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
1048 
1049  // Value Profile Data
1050  uint32_t MDCount = MaxMDCount;
1051  for (auto &VD : VDs) {
1052  Vals.push_back(MDHelper.createConstant(
1053  ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
1054  Vals.push_back(MDHelper.createConstant(
1055  ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
1056  if (--MDCount == 0)
1057  break;
1058  }
1059  Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
1060 }
1061 
1064  uint32_t MaxNumValueData,
1065  InstrProfValueData ValueData[],
1066  uint32_t &ActualNumValueData, uint64_t &TotalC,
1067  bool GetNoICPValue) {
1068  MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1069  if (!MD)
1070  return false;
1071 
1072  unsigned NOps = MD->getNumOperands();
1073 
1074  if (NOps < 5)
1075  return false;
1076 
1077  // Operand 0 is a string tag "VP":
1078  MDString *Tag = cast<MDString>(MD->getOperand(0));
1079  if (!Tag)
1080  return false;
1081 
1082  if (!Tag->getString().equals("VP"))
1083  return false;
1084 
1085  // Now check kind:
1086  ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1087  if (!KindInt)
1088  return false;
1089  if (KindInt->getZExtValue() != ValueKind)
1090  return false;
1091 
1092  // Get total count
1093  ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1094  if (!TotalCInt)
1095  return false;
1096  TotalC = TotalCInt->getZExtValue();
1097 
1098  ActualNumValueData = 0;
1099 
1100  for (unsigned I = 3; I < NOps; I += 2) {
1101  if (ActualNumValueData >= MaxNumValueData)
1102  break;
1103  ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1104  ConstantInt *Count =
1105  mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1106  if (!Value || !Count)
1107  return false;
1108  uint64_t CntValue = Count->getZExtValue();
1109  if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1110  continue;
1111  ValueData[ActualNumValueData].Value = Value->getZExtValue();
1112  ValueData[ActualNumValueData].Count = CntValue;
1113  ActualNumValueData++;
1114  }
1115  return true;
1116 }
1117 
1119  return F.getMetadata(getPGOFuncNameMetadataName());
1120 }
1121 
1123  // Only for internal linkage functions.
1124  if (PGOFuncName == F.getName())
1125  return;
1126  // Don't create duplicated meta-data.
1128  return;
1129  LLVMContext &C = F.getContext();
1130  MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1131  F.setMetadata(getPGOFuncNameMetadataName(), N);
1132 }
1133 
1134 bool needsComdatForCounter(const Function &F, const Module &M) {
1135  if (F.hasComdat())
1136  return true;
1137 
1138  if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1139  return false;
1140 
1141  // See createPGOFuncNameVar for more details. To avoid link errors, profile
1142  // counters for function with available_externally linkage needs to be changed
1143  // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1144  // created. Without using comdat, duplicate entries won't be removed by the
1145  // linker leading to increased data segement size and raw profile size. Even
1146  // worse, since the referenced counter from profile per-function data object
1147  // will be resolved to the common strong definition, the profile counts for
1148  // available_externally functions will end up being duplicated in raw profile
1149  // data. This can result in distorted profile as the counts of those dups
1150  // will be accumulated by the profile merger.
1151  GlobalValue::LinkageTypes Linkage = F.getLinkage();
1152  if (Linkage != GlobalValue::ExternalWeakLinkage &&
1154  return false;
1155 
1156  return true;
1157 }
1158 
1159 // Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1160 bool isIRPGOFlagSet(const Module *M) {
1161  auto IRInstrVar =
1162  M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1163  if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1164  return false;
1165 
1166  // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1167  // have the decl.
1168  if (IRInstrVar->isDeclaration())
1169  return true;
1170 
1171  // Check if the flag is set.
1172  if (!IRInstrVar->hasInitializer())
1173  return false;
1174 
1175  auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1176  if (!InitVal)
1177  return false;
1178  return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1179 }
1180 
1181 // Check if we can safely rename this Comdat function.
1182 bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1183  if (F.getName().empty())
1184  return false;
1185  if (!needsComdatForCounter(F, *(F.getParent())))
1186  return false;
1187  // Unsafe to rename the address-taken function (which can be used in
1188  // function comparison).
1189  if (CheckAddressTaken && F.hasAddressTaken())
1190  return false;
1191  // Only safe to do if this function may be discarded if it is not used
1192  // in the compilation unit.
1193  if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1194  return false;
1195 
1196  // For AvailableExternallyLinkage functions.
1197  if (!F.hasComdat()) {
1199  return true;
1200  }
1201  return true;
1202 }
1203 
1204 // Create the variable for the profile file name.
1205 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1206  if (InstrProfileOutput.empty())
1207  return;
1208  Constant *ProfileNameConst =
1209  ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1210  GlobalVariable *ProfileNameVar = new GlobalVariable(
1211  M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1212  ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1214  Triple TT(M.getTargetTriple());
1215  if (TT.supportsCOMDAT()) {
1216  ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
1217  ProfileNameVar->setComdat(M.getOrInsertComdat(
1218  StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1219  }
1220 }
1221 
1222 Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1223  const std::string &TestFilename,
1224  bool IsCS) {
1225  auto getProfileSum = [IsCS](const std::string &Filename,
1226  CountSumOrPercent &Sum) -> Error {
1227  auto ReaderOrErr = InstrProfReader::create(Filename);
1228  if (Error E = ReaderOrErr.takeError()) {
1229  return E;
1230  }
1231  auto Reader = std::move(ReaderOrErr.get());
1232  Reader->accumulateCounts(Sum, IsCS);
1233  return Error::success();
1234  };
1235  auto Ret = getProfileSum(BaseFilename, Base);
1236  if (Ret)
1237  return Ret;
1238  Ret = getProfileSum(TestFilename, Test);
1239  if (Ret)
1240  return Ret;
1241  this->BaseFilename = &BaseFilename;
1242  this->TestFilename = &TestFilename;
1243  Valid = true;
1244  return Error::success();
1245 }
1246 
1248  Mismatch.NumEntries += 1;
1249  Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1250  for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1251  if (Test.ValueCounts[I] >= 1.0f)
1252  Mismatch.ValueCounts[I] +=
1253  MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1254  }
1255 }
1256 
1258  Unique.NumEntries += 1;
1259  Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1260  for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1261  if (Test.ValueCounts[I] >= 1.0f)
1262  Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1263  }
1264 }
1265 
1267  if (!Valid)
1268  return;
1269 
1270  const char *EntryName =
1271  (Level == ProgramLevel ? "functions" : "edge counters");
1272  if (Level == ProgramLevel) {
1273  OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1274  << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1275  } else {
1276  OS << "Function level:\n"
1277  << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1278  }
1279 
1280  OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1281  if (Mismatch.NumEntries)
1282  OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1283  << "\n";
1284  if (Unique.NumEntries)
1285  OS << " # of " << EntryName
1286  << " only in test_profile: " << Unique.NumEntries << "\n";
1287 
1288  OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1289  << "\n";
1290  if (Mismatch.NumEntries)
1291  OS << " Mismatched count percentage (Edge): "
1292  << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1293  if (Unique.NumEntries)
1294  OS << " Percentage of Edge profile only in test_profile: "
1295  << format("%.3f%%", Unique.CountSum * 100) << "\n";
1296  OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1297  << "\n"
1298  << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1299  << "\n";
1300 
1301  for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1302  if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1303  continue;
1304  char ProfileKindName[20];
1305  switch (I) {
1306  case IPVK_IndirectCallTarget:
1307  strncpy(ProfileKindName, "IndirectCall", 19);
1308  break;
1309  case IPVK_MemOPSize:
1310  strncpy(ProfileKindName, "MemOP", 19);
1311  break;
1312  default:
1313  snprintf(ProfileKindName, 19, "VP[%d]", I);
1314  break;
1315  }
1316  OS << " " << ProfileKindName
1317  << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1318  << "\n";
1319  if (Mismatch.NumEntries)
1320  OS << " Mismatched count percentage (" << ProfileKindName
1321  << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1322  if (Unique.NumEntries)
1323  OS << " Percentage of " << ProfileKindName
1324  << " profile only in test_profile: "
1325  << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1326  OS << " " << ProfileKindName
1327  << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1328  << "\n"
1329  << " " << ProfileKindName
1330  << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1331  << "\n";
1332  }
1333 }
1334 
1335 namespace IndexedInstrProf {
1336 // A C++14 compatible version of the offsetof macro.
1337 template <typename T1, typename T2>
1338 inline size_t constexpr offsetOf(T1 T2::*Member) {
1339  constexpr T2 Object{};
1340  return size_t(&(Object.*Member)) - size_t(&Object);
1341 }
1342 
1343 static inline uint64_t read(const unsigned char *Buffer, size_t Offset) {
1344  return *reinterpret_cast<const uint64_t *>(Buffer + Offset);
1345 }
1346 
1348  using namespace support;
1349  return endian::byte_swap<uint64_t, little>(Version);
1350 }
1351 
1352 Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
1353  using namespace support;
1354  static_assert(std::is_standard_layout_v<Header>,
1355  "The header should be standard layout type since we use offset "
1356  "of fields to read.");
1357  Header H;
1358 
1359  H.Magic = read(Buffer, offsetOf(&Header::Magic));
1360  // Check the magic number.
1361  uint64_t Magic = endian::byte_swap<uint64_t, little>(H.Magic);
1363  return make_error<InstrProfError>(instrprof_error::bad_magic);
1364 
1365  // Read the version.
1366  H.Version = read(Buffer, offsetOf(&Header::Version));
1367  if (GET_VERSION(H.formatVersion()) >
1369  return make_error<InstrProfError>(instrprof_error::unsupported_version);
1370 
1371  switch (GET_VERSION(H.formatVersion())) {
1372  // When a new field is added in the header add a case statement here to
1373  // populate it.
1374  static_assert(
1376  "Please update the reading code below if a new field has been added, "
1377  "if not add a case statement to fall through to the latest version.");
1378  case 8ull:
1379  H.MemProfOffset = read(Buffer, offsetOf(&Header::MemProfOffset));
1380  [[fallthrough]];
1381  default: // Version7 (when the backwards compatible header was introduced).
1382  H.HashType = read(Buffer, offsetOf(&Header::HashType));
1383  H.HashOffset = read(Buffer, offsetOf(&Header::HashOffset));
1384  }
1385 
1386  return H;
1387 }
1388 
1389 size_t Header::size() const {
1390  switch (GET_VERSION(formatVersion())) {
1391  // When a new field is added to the header add a case statement here to
1392  // compute the size as offset of the new field + size of the new field. This
1393  // relies on the field being added to the end of the list.
1395  "Please update the size computation below if a new field has "
1396  "been added to the header, if not add a case statement to "
1397  "fall through to the latest version.");
1398  case 8ull:
1400  default: // Version7 (when the backwards compatible header was introduced).
1401  return offsetOf(&Header::HashOffset) + sizeof(Header::HashOffset);
1402  }
1403 }
1404 
1405 } // namespace IndexedInstrProf
1406 
1407 } // end namespace llvm
llvm::SaturatingMultiply
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:761
llvm::OverlapStats::Unique
CountSumOrPercent Unique
Definition: InstrProf.h:659
llvm::OverlapStats::Level
OverlapStatsLevel Level
Definition: InstrProf.h:660
llvm::createPGOFuncNameMetadata
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1122
Compression.h
llvm::OverlapStats::score
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:683
llvm::allocValueProfData
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
Definition: InstrProf.cpp:931
llvm::compression::zlib::decompress
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
Definition: Compression.cpp:123
llvm::instrprof_error::bad_header
@ bad_header
MathExtras.h
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: Globals.cpp:189
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::MDBuilder::createConstant
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
Definition: MDBuilder.cpp:24
llvm::msgpack::Endianness
constexpr support::endianness Endianness
The endianness of all multi-byte encoded values in MessagePack.
Definition: MsgPack.h:24
llvm::InstrProfRecord::PseudoWarm
@ PseudoWarm
Definition: InstrProf.h:830
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::IndexedInstrProf::Header::Version
uint64_t Version
Definition: InstrProf.h:1066
llvm::GlobalValue::getGlobalIdentifier
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:161
llvm::GlobalValue::LinkOnceAnyLinkage
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
Definition: GlobalValue.h:50
Metadata.h
llvm::OverlapStats::Valid
bool Valid
Definition: InstrProf.h:665
llvm::InstrProfError::message
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:256
llvm::getNumValueKindsInstrProf
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
Definition: InstrProf.cpp:810
llvm::instrprof_error::uncompress_failed
@ uncompress_failed
llvm::InstrProfRecord::getCountPseudoKind
CountPseudoKind getCountPseudoKind() const
Definition: InstrProf.h:836
llvm::GlobalValue::HiddenVisibility
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
llvm::Function
Definition: Function.h:60
StringRef.h
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::IndexedInstrProf::offsetOf
constexpr size_t offsetOf(T1 T2::*Member)
Definition: InstrProf.cpp:1338
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:629
llvm::InstrProfRecord::getValueForSite
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
Definition: InstrProf.h:955
llvm::needsComdatForCounter
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1134
name
static const char * name
Definition: SMEABIPass.cpp:49
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:476
llvm::SmallVector< uint8_t, 128 >
llvm::InstrProfRecord::setPseudoCount
void setPseudoCount(CountPseudoKind Kind)
Definition: InstrProf.h:844
Path.h
llvm::GlobalValue::isLocalLinkage
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:404
llvm::getPGOFuncName
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Definition: InstrProf.cpp:300
ErrorHandling.h
llvm::InstrProfSymtab::create
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
llvm::OverlapStats::dump
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1266
llvm::Error::success
static ErrorSuccess success()
Create a success value.
Definition: Error.h:329
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::getInstrProfNameSeparator
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:169
llvm::IndexedInstrProf::Header::size
size_t size() const
Definition: InstrProf.cpp:1389
llvm::getValueForSiteInstrProf
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
Definition: InstrProf.cpp:830
Error.h
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
SwapByteOrder.h
llvm::createPGOFuncNameVar
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:379
llvm::tgtok::VarName
@ VarName
Definition: TGLexer.h:72
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
StaticFuncFullModulePrefix
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
llvm::getInstrMaxCountValue
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition: InstrProf.h:64
llvm::instrprof_error::zlib_unavailable
@ zlib_unavailable
llvm::OverlapStats
Definition: InstrProf.h:650
Module.h
llvm::InstrProfValueSiteRecord::scale
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:677
llvm::InstrProfRecord::overlap
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:604
T1
#define T1
Definition: Mips16ISelLowering.cpp:340
llvm::instrprof_error::value_site_count_mismatch
@ value_site_count_mismatch
T
#define T
Definition: Mips16ISelLowering.cpp:341
llvm::InstrProfRecord::PseudoHot
@ PseudoHot
Definition: InstrProf.h:829
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::IndexedInstrProf::Version8
@ Version8
Definition: InstrProf.h:1052
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:119
llvm::Expected
Tagged union holding either a T or a Error.
Definition: APFloat.h:41
llvm::GlobalValue::LinkageTypes
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
llvm::getPGOFuncNameVarName
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:334
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1205
llvm::instrprof_error::invalid_prof
@ invalid_prof
llvm::dwarf::Tag
Tag
Definition: Dwarf.h:105
LLVM_ATTRIBUTE_UNUSED
#define LLVM_ATTRIBUTE_UNUSED
Definition: Compiler.h:172
llvm::IndexedInstrProf::Header::MemProfOffset
uint64_t MemProfOffset
Definition: InstrProf.h:1070
llvm::GlobalValue::isDiscardableIfUnused
bool isDiscardableIfUnused() const
Definition: GlobalValue.h:543
llvm::consumeError
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1042
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:239
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:159
llvm::InstrProfRecord
Profiling information for a single function.
Definition: InstrProf.h:730
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1400
F
#define F(x, y, z)
Definition: MD5.cpp:55
InstrProfData.inc
llvm::Instruction::setMetadata
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1456
llvm::InstrProfValueSiteRecord::merge
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:655
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1298
llvm::instrprof_error::unexpected_debug_info_for_correlation
@ unexpected_debug_info_for_correlation
llvm::OverlapStats::TestFilename
const std::string * TestFilename
Definition: InstrProf.h:662
getInstrProfErrString
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Definition: InstrProf.cpp:77
Instruction.h
CommandLine.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:78
llvm::decodeULEB128
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
Definition: LEB128.h:128
llvm::coverage::CurrentVersion
@ CurrentVersion
Definition: CoverageMapping.h:1009
llvm::compression::zlib::BestSizeCompression
constexpr int BestSizeCompression
Definition: Compression.h:39
GlobalValue.h
llvm::InstrProfRecord::getNumValueDataForSite
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:949
llvm::MutableArrayRef
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:28
llvm::StringRef::startswith
bool startswith(StringRef Prefix) const
Definition: StringRef.h:260
ptrdiff_t
Constants.h
llvm::CountSumOrPercent::CountSum
double CountSum
Definition: InstrProf.h:638
SmallString.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::support::little
@ little
Definition: Endian.h:27
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::InstrProfRecordClosure
static ValueProfRecordClosure InstrProfRecordClosure
Definition: InstrProf.cpp:842
llvm::canRenameComdatFunc
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1182
llvm::instrprof_error::compress_failed
@ compress_failed
llvm::InstrProfRecord::getNumValueSites
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:945
llvm::allocValueProfDataInstrProf
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
Definition: InstrProf.cpp:835
llvm::Triple::supportsCOMDAT
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:942
llvm::stripDirPrefix
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
Definition: InstrProf.cpp:272
llvm::InstrProfError::ID
static char ID
Definition: InstrProf.h:367
llvm::dwarf::toStringRef
StringRef toStringRef(const Optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
Definition: DWARFFormValue.h:193
llvm::Instruction
Definition: Instruction.h:42
InstrProf.h
llvm::swapToHostOrder
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig)
Definition: InstrProf.cpp:922
llvm::instrprof_error::truncated
@ truncated
MDBuilder.h
llvm::IndexedInstrProf::Header::Magic
uint64_t Magic
Definition: InstrProf.h:1065
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::IndexedInstrProf::Version
const uint64_t Version
Definition: InstrProf.h:1056
llvm::OverlapStats::FuncName
StringRef FuncName
Definition: InstrProf.h:663
StaticFuncStripDirNamePrefix
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
llvm::IndexedInstrProf::Header
Definition: InstrProf.h:1064
llvm::instrprof_error::bad_magic
@ bad_magic
llvm::IndexedInstrProf::Header::readFromBuffer
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1352
llvm::instrprof_error::unsupported_version
@ unsupported_version
llvm::compression::zlib::isAvailable
bool isAvailable()
Definition: Compression.cpp:105
Type.h
llvm::instrprof_error::too_large
@ too_large
llvm::InstrProfSectKind
InstrProfSectKind
Definition: InstrProf.h:58
llvm::Instruction::getMetadata
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:276
llvm::ARM_PROC::IE
@ IE
Definition: ARMBaseInfo.h:27
llvm::OutputFileType::Object
@ Object
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1292
llvm::MDBuilder::createString
MDString * createString(StringRef Str)
Return the given string as metadata.
Definition: MDBuilder.cpp:20
llvm::getHostEndianness
support::endianness getHostEndianness()
Definition: InstrProf.h:991
llvm::StringRef::empty
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
llvm::getInstrProfNameVarPrefix
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:90
llvm::instrprof_error::unsupported_hash_type
@ unsupported_hash_type
llvm::cl::opt< bool >
InstrProfReader.h
llvm::CountSumOrPercent::NumEntries
uint64_t NumEntries
Definition: InstrProf.h:637
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:135
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::count
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1895
uint64_t
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::GlobalValue::WeakAnyLinkage
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
llvm::SoftInstrProfErrors::addError
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
Definition: InstrProf.cpp:231
llvm::getNumValueSitesInstrProf
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:814
LEB128.h
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
llvm::InstrProfRecord::NotPseudo
@ NotPseudo
Definition: InstrProf.h:828
llvm::CountSumOrPercent
Definition: InstrProf.h:636
llvm::isIRPGOFlagSet
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1160
llvm::instrprof_error::empty_raw_profile
@ empty_raw_profile
I
#define I(x, y, z)
Definition: MD5.cpp:58
StringExtras.h
llvm::instrprof_error
instrprof_error
Definition: InstrProf.h:308
llvm::instrprof_error::success
@ success
llvm::SaturatingMultiplyAdd
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition: MathExtras.h:807
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
ArrayRef.h
llvm::MDString::get
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:498
message
message(STATUS "Targeting ${t}") add_subdirectory($
Definition: CMakeLists.txt:34
llvm::IndexedInstrProf::Header::HashType
uint64_t HashType
Definition: InstrProf.h:1068
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::instrprof_error::unknown_function
@ unknown_function
memcpy
<%struct.s * > cast struct s *S to sbyte *< sbyte * > sbyte uint cast struct s *agg result to sbyte *< sbyte * > sbyte uint cast struct s *memtmp to sbyte *< sbyte * > sbyte uint ret void llc ends up issuing two memcpy or custom lower memcpy(of small size) to be ldmia/stmia. I think option 2 is better but the current register allocator cannot allocate a chunk of registers at a time. A feasible temporary solution is to use specific physical registers at the lowering time for small(<
llvm::instrprof_error::unrecognized_format
@ unrecognized_format
llvm::Record
Definition: Record.h:1573
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::InstrProfValueKind
InstrProfValueKind
Definition: InstrProf.h:244
llvm::MDNode
Metadata node.
Definition: Metadata.h:944
llvm::collectPGOFuncNameStrings
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:438
llvm::getNumValueDataInstrProf
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:819
llvm::StringRef::bytes_begin
const unsigned char * bytes_begin() const
Definition: StringRef.h:115
llvm::InstrProfSymtab::getFunctionHashFromAddress
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:424
Triple.h
llvm::InstrProfRecord::accumulateCounts
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:535
llvm::OverlapStats::addOneUnique
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1257
llvm::ArrayRef< std::string >
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::IndexedInstrProf::Magic
const uint64_t Magic
Definition: InstrProf.h:1025
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
uint32_t
llvm::instrprof_error::eof
@ eof
Compiler.h
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2967
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::format
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:124
llvm::GlobalValue::AvailableExternallyLinkage
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
llvm::instrprof_error::hash_mismatch
@ hash_mismatch
llvm::Triple::ObjectFormatType
ObjectFormatType
Definition: Triple.h:278
llvm::ValueMap
See the file comment.
Definition: ValueMap.h:85
llvm::InstrProfValueSiteRecord::ValueData
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:700
llvm::raw_fd_ostream
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:441
llvm::instrprof_category
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:179
llvm::annotateValueSite
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1017
llvm::AMDGPU::SendMsg::Msg
const CustomOperand< const MCSubtargetInfo & > Msg[]
Definition: AMDGPUAsmUtils.cpp:39
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
llvm::InstrProfRecord::merge
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:708
llvm::DoInstrProfNameCompression
cl::opt< bool > DoInstrProfNameCompression
llvm::InstrProfRecord::overlapValueProfData
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:586
llvm::StringRef::size
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
llvm::readPGOFuncNameStrings
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
Definition: InstrProf.cpp:493
llvm::partition_point
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
Definition: STLExtras.h:1960
Constant.h
llvm::IndexedInstrProf::Header::formatVersion
uint64_t formatVersion() const
Definition: InstrProf.cpp:1347
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
llvm::InstrProfSymtab
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:446
H
#define H(x, y, z)
Definition: MD5.cpp:57
llvm::InstrProfValueSiteRecord::overlap
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:555
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:152
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:155
GlobalVariable.h
llvm::compression::zlib::compress
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Definition: Compression.cpp:107
llvm::GlobalValue::getGUID
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:591
Casting.h
llvm::Triple::MachO
@ MachO
Definition: Triple.h:285
Function.h
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::OverlapStats::ProgramLevel
@ ProgramLevel
Definition: InstrProf.h:651
llvm::InstrProfReader::create
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, const InstrProfCorrelator *Correlator=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
Definition: InstrProfReader.cpp:79
llvm::instrprof_error::count_mismatch
@ count_mismatch
llvm::NOMORE_ICP_MAGICNUM
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
Definition: Metadata.h:57
llvm::OverlapStats::Overlap
CountSumOrPercent Overlap
Definition: InstrProf.h:657
llvm::getNumValueDataForSiteInstrProf
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
Definition: InstrProf.cpp:824
llvm::OverlapStats::Mismatch
CountSumOrPercent Mismatch
Definition: InstrProf.h:658
llvm::getPGOFuncNameMetadataName
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:272
llvm::InstrProfRecord::addValueData
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:788
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:475
llvm::MDBuilder
Definition: MDBuilder.h:36
llvm::IndexedInstrProf::Header::HashOffset
uint64_t HashOffset
Definition: InstrProf.h:1069
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:212
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
llvm::sys::path::is_separator
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
Definition: Path.cpp:601
llvm::InstrProfValueSiteRecord
Definition: InstrProf.h:698
llvm::InstrProfSymtab::addFuncName
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:507
llvm::Triple::COFF
@ COFF
Definition: Triple.h:281
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
SmallVector.h
llvm::OverlapStats::FuncHash
uint64_t FuncHash
Definition: InstrProf.h:664
N
#define N
llvm::OverlapStats::BaseFilename
const std::string * BaseFilename
Definition: InstrProf.h:661
llvm::instrprof_error::unable_to_correlate_profile
@ unable_to_correlate_profile
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:164
support
Reimplement select in terms of SEL *We would really like to support but we need to prove that the add doesn t need to overflow between the two bit chunks *Implement pre post increment support(e.g. PR935) *Implement smarter const ant generation for binops with large immediates. A few ARMv6T2 ops should be pattern matched
Definition: README.txt:10
llvm::encodeULEB128
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
llvm::support::endianness
endianness
Definition: Endian.h:27
llvm::OverlapStats::Base
CountSumOrPercent Base
Definition: InstrProf.h:653
llvm::InstrProfValueSiteRecord::sortByTargetValues
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:708
llvm::instrprof_error::malformed
@ malformed
llvm::CountSumOrPercent::ValueCounts
double ValueCounts[IPVK_Last - IPVK_First+1]
Definition: InstrProf.h:639
llvm::getFuncNameWithoutPrefix
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:323
llvm::OverlapStats::accumulateCounts
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1222
llvm::InstrProfRecord::CountPseudoKind
CountPseudoKind
Definition: InstrProf.h:827
llvm::IndexedInstrProf::read
static uint64_t read(const unsigned char *Buffer, size_t Offset)
Definition: InstrProf.cpp:1343
llvm::getValueProfDataFromInst
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1062
LLVMContext.h
llvm::cl::desc
Definition: CommandLine.h:413
llvm::GlobalValue::ExternalWeakLinkage
@ ExternalWeakLinkage
ExternalWeak linkage description.
Definition: GlobalValue.h:57
Endian.h
llvm::MDString
A single uniqued string.
Definition: Metadata.h:612
llvm::GlobalValue::LinkOnceODRLinkage
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
llvm::raw_string_ostream::str
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:647
llvm::GlobalValue::setVisibility
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:250
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:692
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::OverlapStats::addOneMismatch
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1247
llvm::InstrProfRecord::Counts
std::vector< uint64_t > Counts
Definition: InstrProf.h:731
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:153
llvm::instrprof_error::counter_overflow
@ counter_overflow
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition: StringRef.h:601
llvm::OverlapStats::Test
CountSumOrPercent Test
Definition: InstrProf.h:655
llvm::instrprof_error::missing_debug_info_for_correlation
@ missing_debug_info_for_correlation
Other
Optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1251
llvm::getPGOFuncNameMetadata
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1118
llvm::StringRef::bytes_end
const unsigned char * bytes_end() const
Definition: StringRef.h:118
llvm::InstrProfRecord::scale
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:759