LLVM  14.0.0git
InstrProf.cpp
Go to the documentation of this file.
1 //===- InstrProf.cpp - Instrumented profiling format support --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains support for clang's instrumentation based PGO and
10 // coverage.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SmallString.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/Triple.h"
21 #include "llvm/Config/config.h"
22 #include "llvm/IR/Constant.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/GlobalValue.h"
26 #include "llvm/IR/GlobalVariable.h"
27 #include "llvm/IR/Instruction.h"
28 #include "llvm/IR/LLVMContext.h"
29 #include "llvm/IR/MDBuilder.h"
30 #include "llvm/IR/Metadata.h"
31 #include "llvm/IR/Module.h"
32 #include "llvm/IR/Type.h"
34 #include "llvm/Support/Casting.h"
36 #include "llvm/Support/Compiler.h"
38 #include "llvm/Support/Endian.h"
39 #include "llvm/Support/Error.h"
41 #include "llvm/Support/LEB128.h"
44 #include "llvm/Support/Path.h"
46 #include <algorithm>
47 #include <cassert>
48 #include <cstddef>
49 #include <cstdint>
50 #include <cstring>
51 #include <memory>
52 #include <string>
53 #include <system_error>
54 #include <utility>
55 #include <vector>
56 
57 using namespace llvm;
58 
60  "static-func-full-module-prefix", cl::init(true), cl::Hidden,
61  cl::desc("Use full module build paths in the profile counter names for "
62  "static functions."));
63 
64 // This option is tailored to users that have different top-level directory in
65 // profile-gen and profile-use compilation. Users need to specific the number
66 // of levels to strip. A value larger than the number of directories in the
67 // source file will strip all the directory names and only leave the basename.
68 //
69 // Note current ThinLTO module importing for the indirect-calls assumes
70 // the source directory name not being stripped. A non-zero option value here
71 // can potentially prevent some inter-module indirect-call-promotions.
73  "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
74  cl::desc("Strip specified level of directory name from source path in "
75  "the profile counter name for static functions."));
76 
77 static std::string getInstrProfErrString(instrprof_error Err,
78  const std::string &ErrMsg = "") {
79  std::string Msg;
80  raw_string_ostream OS(Msg);
81 
82  switch (Err) {
84  OS << "success";
85  break;
87  OS << "end of File";
88  break;
90  OS << "unrecognized instrumentation profile encoding format";
91  break;
93  OS << "invalid instrumentation profile data (bad magic)";
94  break;
96  OS << "invalid instrumentation profile data (file header is corrupt)";
97  break;
99  OS << "unsupported instrumentation profile format version";
100  break;
102  OS << "unsupported instrumentation profile hash type";
103  break;
105  OS << "too much profile data";
106  break;
108  OS << "truncated profile data";
109  break;
111  OS << "malformed instrumentation profile data";
112  break;
114  OS << "debug info for correlation is required";
115  break;
117  OS << "debug info for correlation is not necessary";
118  break;
120  OS << "unable to correlate profile";
121  break;
123  OS << "invalid profile created. Please file a bug "
124  "at: " BUG_REPORT_URL
125  " and include the profraw files that caused this error.";
126  break;
128  OS << "no profile data available for function";
129  break;
131  OS << "function control flow change detected (hash mismatch)";
132  break;
134  OS << "function basic block count change detected (counter mismatch)";
135  break;
137  OS << "counter overflow";
138  break;
140  OS << "function value site count change detected (counter mismatch)";
141  break;
143  OS << "failed to compress data (zlib)";
144  break;
146  OS << "failed to uncompress data (zlib)";
147  break;
149  OS << "empty raw profile file";
150  break;
152  OS << "profile uses zlib compression but the profile reader was built "
153  "without zlib support";
154  break;
155  }
156 
157  // If optional error message is not empty, append it to the message.
158  if (!ErrMsg.empty())
159  OS << ": " << ErrMsg;
160 
161  return OS.str();
162 }
163 
164 namespace {
165 
166 // FIXME: This class is only here to support the transition to llvm::Error. It
167 // will be removed once this transition is complete. Clients should prefer to
168 // deal with the Error value directly, rather than converting to error_code.
169 class InstrProfErrorCategoryType : public std::error_category {
170  const char *name() const noexcept override { return "llvm.instrprof"; }
171 
172  std::string message(int IE) const override {
173  return getInstrProfErrString(static_cast<instrprof_error>(IE));
174  }
175 };
176 
177 } // end anonymous namespace
178 
180 
182  return *ErrorCategory;
183 }
184 
185 namespace {
186 
187 const char *InstrProfSectNameCommon[] = {
188 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
189  SectNameCommon,
191 };
192 
193 const char *InstrProfSectNameCoff[] = {
194 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
195  SectNameCoff,
197 };
198 
199 const char *InstrProfSectNamePrefix[] = {
200 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
201  Prefix,
203 };
204 
205 } // namespace
206 
207 namespace llvm {
208 
210  "enable-name-compression",
211  cl::desc("Enable name/filename string compression"), cl::init(true));
212 
215  bool AddSegmentInfo) {
216  std::string SectName;
217 
218  if (OF == Triple::MachO && AddSegmentInfo)
219  SectName = InstrProfSectNamePrefix[IPSK];
220 
221  if (OF == Triple::COFF)
222  SectName += InstrProfSectNameCoff[IPSK];
223  else
224  SectName += InstrProfSectNameCommon[IPSK];
225 
226  if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
227  SectName += ",regular,live_support";
228 
229  return SectName;
230 }
231 
234  return;
235 
236  if (FirstError == instrprof_error::success)
237  FirstError = IE;
238 
239  switch (IE) {
241  ++NumHashMismatches;
242  break;
244  ++NumCountMismatches;
245  break;
247  ++NumCounterOverflows;
248  break;
250  ++NumValueSiteCountMismatches;
251  break;
252  default:
253  llvm_unreachable("Not a soft error");
254  }
255 }
256 
257 std::string InstrProfError::message() const {
258  return getInstrProfErrString(Err, Msg);
259 }
260 
261 char InstrProfError::ID = 0;
262 
263 std::string getPGOFuncName(StringRef RawFuncName,
265  StringRef FileName,
267  return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName);
268 }
269 
270 // Strip NumPrefix level of directory name from PathNameStr. If the number of
271 // directory separators is less than NumPrefix, strip all the directories and
272 // leave base file name only.
273 static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
274  uint32_t Count = NumPrefix;
275  uint32_t Pos = 0, LastPos = 0;
276  for (auto & CI : PathNameStr) {
277  ++Pos;
279  LastPos = Pos;
280  --Count;
281  }
282  if (Count == 0)
283  break;
284  }
285  return PathNameStr.substr(LastPos);
286 }
287 
288 // Return the PGOFuncName. This function has some special handling when called
289 // in LTO optimization. The following only applies when calling in LTO passes
290 // (when \c InLTO is true): LTO's internalization privatizes many global linkage
291 // symbols. This happens after value profile annotation, but those internal
292 // linkage functions should not have a source prefix.
293 // Additionally, for ThinLTO mode, exported internal functions are promoted
294 // and renamed. We need to ensure that the original internal PGO name is
295 // used when computing the GUID that is compared against the profiled GUIDs.
296 // To differentiate compiler generated internal symbols from original ones,
297 // PGOFuncName meta data are created and attached to the original internal
298 // symbols in the value profile annotation step
299 // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
300 // data, its original linkage must be non-internal.
301 std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
302  if (!InLTO) {
303  StringRef FileName(F.getParent()->getSourceFileName());
304  uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
305  if (StripLevel < StaticFuncStripDirNamePrefix)
306  StripLevel = StaticFuncStripDirNamePrefix;
307  if (StripLevel)
308  FileName = stripDirPrefix(FileName, StripLevel);
309  return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
310  }
311 
312  // In LTO mode (when InLTO is true), first check if there is a meta data.
313  if (MDNode *MD = getPGOFuncNameMetadata(F)) {
314  StringRef S = cast<MDString>(MD->getOperand(0))->getString();
315  return S.str();
316  }
317 
318  // If there is no meta data, the function must be a global before the value
319  // profile annotation pass. Its current linkage may be internal if it is
320  // internalized in LTO mode.
321  return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
322 }
323 
325  if (FileName.empty())
326  return PGOFuncName;
327  // Drop the file name including ':'. See also getPGOFuncName.
328  if (PGOFuncName.startswith(FileName))
329  PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
330  return PGOFuncName;
331 }
332 
333 // \p FuncName is the string used as profile lookup key for the function. A
334 // symbol is created to hold the name. Return the legalized symbol name.
335 std::string getPGOFuncNameVarName(StringRef FuncName,
337  std::string VarName = std::string(getInstrProfNameVarPrefix());
338  VarName += FuncName;
339 
341  return VarName;
342 
343  // Now fix up illegal chars in local VarName that may upset the assembler.
344  const char *InvalidChars = "-:<>/\"'";
345  size_t found = VarName.find_first_of(InvalidChars);
346  while (found != std::string::npos) {
347  VarName[found] = '_';
348  found = VarName.find_first_of(InvalidChars, found + 1);
349  }
350  return VarName;
351 }
352 
355  StringRef PGOFuncName) {
356  // We generally want to match the function's linkage, but available_externally
357  // and extern_weak both have the wrong semantics, and anything that doesn't
358  // need to link across compilation units doesn't need to be visible at all.
366 
367  auto *Value =
368  ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
369  auto FuncNameVar =
370  new GlobalVariable(M, Value->getType(), true, Linkage, Value,
371  getPGOFuncNameVarName(PGOFuncName, Linkage));
372 
373  // Hide the symbol so that we correctly get a copy for each executable.
374  if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
375  FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
376 
377  return FuncNameVar;
378 }
379 
381  return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
382 }
383 
385  for (Function &F : M) {
386  // Function may not have a name: like using asm("") to overwrite the name.
387  // Ignore in this case.
388  if (!F.hasName())
389  continue;
390  const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
391  if (Error E = addFuncName(PGOFuncName))
392  return E;
393  MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
394  // In ThinLTO, local function may have been promoted to global and have
395  // suffix ".llvm." added to the function name. We need to add the
396  // stripped function name to the symbol table so that we can find a match
397  // from profile.
398  //
399  // We may have other suffixes similar as ".llvm." which are needed to
400  // be stripped before the matching, but ".__uniq." suffix which is used
401  // to differentiate internal linkage functions in different modules
402  // should be kept. Now this is the only suffix with the pattern ".xxx"
403  // which is kept before matching.
404  const std::string UniqSuffix = ".__uniq.";
405  auto pos = PGOFuncName.find(UniqSuffix);
406  // Search '.' after ".__uniq." if ".__uniq." exists, otherwise
407  // search '.' from the beginning.
408  if (pos != std::string::npos)
409  pos += UniqSuffix.length();
410  else
411  pos = 0;
412  pos = PGOFuncName.find('.', pos);
413  if (pos != std::string::npos && pos != 0) {
414  const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
415  if (Error E = addFuncName(OtherFuncName))
416  return E;
417  MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
418  }
419  }
420  Sorted = false;
421  finalizeSymtab();
422  return Error::success();
423 }
424 
426  finalizeSymtab();
427  auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
428  return A.first < Address;
429  });
430  // Raw function pointer collected by value profiler may be from
431  // external functions that are not instrumented. They won't have
432  // mapping data to be used by the deserializer. Force the value to
433  // be 0 in this case.
434  if (It != AddrToMD5Map.end() && It->first == Address)
435  return (uint64_t)It->second;
436  return 0;
437 }
438 
440  bool doCompression, std::string &Result) {
441  assert(!NameStrs.empty() && "No name data to emit");
442 
443  uint8_t Header[16], *P = Header;
444  std::string UncompressedNameStrings =
445  join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
446 
447  assert(StringRef(UncompressedNameStrings)
448  .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
449  "PGO name is invalid (contains separator token)");
450 
451  unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
452  P += EncLen;
453 
454  auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
455  EncLen = encodeULEB128(CompressedLen, P);
456  P += EncLen;
457  char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
458  unsigned HeaderLen = P - &Header[0];
459  Result.append(HeaderStr, HeaderLen);
460  Result += InputStr;
461  return Error::success();
462  };
463 
464  if (!doCompression) {
465  return WriteStringToResult(0, UncompressedNameStrings);
466  }
467 
468  SmallString<128> CompressedNameStrings;
469  Error E = zlib::compress(StringRef(UncompressedNameStrings),
470  CompressedNameStrings, zlib::BestSizeCompression);
471  if (E) {
473  return make_error<InstrProfError>(instrprof_error::compress_failed);
474  }
475 
476  return WriteStringToResult(CompressedNameStrings.size(),
477  CompressedNameStrings);
478 }
479 
481  auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
482  StringRef NameStr =
483  Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
484  return NameStr;
485 }
486 
488  std::string &Result, bool doCompression) {
489  std::vector<std::string> NameStrs;
490  for (auto *NameVar : NameVars) {
491  NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
492  }
494  NameStrs, zlib::isAvailable() && doCompression, Result);
495 }
496 
498  const uint8_t *P = NameStrings.bytes_begin();
499  const uint8_t *EndP = NameStrings.bytes_end();
500  while (P < EndP) {
501  uint32_t N;
502  uint64_t UncompressedSize = decodeULEB128(P, &N);
503  P += N;
504  uint64_t CompressedSize = decodeULEB128(P, &N);
505  P += N;
506  bool isCompressed = (CompressedSize != 0);
507  SmallString<128> UncompressedNameStrings;
508  StringRef NameStrings;
509  if (isCompressed) {
511  return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
512 
513  StringRef CompressedNameStrings(reinterpret_cast<const char *>(P),
514  CompressedSize);
515  if (Error E =
516  zlib::uncompress(CompressedNameStrings, UncompressedNameStrings,
517  UncompressedSize)) {
519  return make_error<InstrProfError>(instrprof_error::uncompress_failed);
520  }
521  P += CompressedSize;
522  NameStrings = StringRef(UncompressedNameStrings.data(),
523  UncompressedNameStrings.size());
524  } else {
525  NameStrings =
526  StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
527  P += UncompressedSize;
528  }
529  // Now parse the name strings.
531  NameStrings.split(Names, getInstrProfNameSeparator());
532  for (StringRef &Name : Names)
533  if (Error E = Symtab.addFuncName(Name))
534  return E;
535 
536  while (P < EndP && *P == 0)
537  P++;
538  }
539  return Error::success();
540 }
541 
543  uint64_t FuncSum = 0;
544  Sum.NumEntries += Counts.size();
545  for (uint64_t Count : Counts)
546  FuncSum += Count;
547  Sum.CountSum += FuncSum;
548 
549  for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
550  uint64_t KindSum = 0;
551  uint32_t NumValueSites = getNumValueSites(VK);
552  for (size_t I = 0; I < NumValueSites; ++I) {
554  std::unique_ptr<InstrProfValueData[]> VD = getValueForSite(VK, I);
555  for (uint32_t V = 0; V < NV; V++)
556  KindSum += VD[V].Count;
557  }
558  Sum.ValueCounts[VK] += KindSum;
559  }
560 }
561 
564  OverlapStats &Overlap,
565  OverlapStats &FuncLevelOverlap) {
566  this->sortByTargetValues();
567  Input.sortByTargetValues();
568  double Score = 0.0f, FuncLevelScore = 0.0f;
569  auto I = ValueData.begin();
570  auto IE = ValueData.end();
571  auto J = Input.ValueData.begin();
572  auto JE = Input.ValueData.end();
573  while (I != IE && J != JE) {
574  if (I->Value == J->Value) {
575  Score += OverlapStats::score(I->Count, J->Count,
576  Overlap.Base.ValueCounts[ValueKind],
577  Overlap.Test.ValueCounts[ValueKind]);
578  FuncLevelScore += OverlapStats::score(
579  I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
580  FuncLevelOverlap.Test.ValueCounts[ValueKind]);
581  ++I;
582  } else if (I->Value < J->Value) {
583  ++I;
584  continue;
585  }
586  ++J;
587  }
588  Overlap.Overlap.ValueCounts[ValueKind] += Score;
589  FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
590 }
591 
592 // Return false on mismatch.
594  InstrProfRecord &Other,
595  OverlapStats &Overlap,
596  OverlapStats &FuncLevelOverlap) {
597  uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
598  assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));
599  if (!ThisNumValueSites)
600  return;
601 
602  std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
603  getOrCreateValueSitesForKind(ValueKind);
605  Other.getValueSitesForKind(ValueKind);
606  for (uint32_t I = 0; I < ThisNumValueSites; I++)
607  ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
608  FuncLevelOverlap);
609 }
610 
612  OverlapStats &FuncLevelOverlap,
613  uint64_t ValueCutoff) {
614  // FuncLevel CountSum for other should already computed and nonzero.
615  assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
616  accumulateCounts(FuncLevelOverlap.Base);
617  bool Mismatch = (Counts.size() != Other.Counts.size());
618 
619  // Check if the value profiles mismatch.
620  if (!Mismatch) {
621  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
622  uint32_t ThisNumValueSites = getNumValueSites(Kind);
623  uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
624  if (ThisNumValueSites != OtherNumValueSites) {
625  Mismatch = true;
626  break;
627  }
628  }
629  }
630  if (Mismatch) {
631  Overlap.addOneMismatch(FuncLevelOverlap.Test);
632  return;
633  }
634 
635  // Compute overlap for value counts.
636  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
637  overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
638 
639  double Score = 0.0;
640  uint64_t MaxCount = 0;
641  // Compute overlap for edge counts.
642  for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
643  Score += OverlapStats::score(Counts[I], Other.Counts[I],
644  Overlap.Base.CountSum, Overlap.Test.CountSum);
645  MaxCount = std::max(Other.Counts[I], MaxCount);
646  }
647  Overlap.Overlap.CountSum += Score;
648  Overlap.Overlap.NumEntries += 1;
649 
650  if (MaxCount >= ValueCutoff) {
651  double FuncScore = 0.0;
652  for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
653  FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
654  FuncLevelOverlap.Base.CountSum,
655  FuncLevelOverlap.Test.CountSum);
656  FuncLevelOverlap.Overlap.CountSum = FuncScore;
657  FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
658  FuncLevelOverlap.Valid = true;
659  }
660 }
661 
663  uint64_t Weight,
664  function_ref<void(instrprof_error)> Warn) {
665  this->sortByTargetValues();
666  Input.sortByTargetValues();
667  auto I = ValueData.begin();
668  auto IE = ValueData.end();
669  for (const InstrProfValueData &J : Input.ValueData) {
670  while (I != IE && I->Value < J.Value)
671  ++I;
672  if (I != IE && I->Value == J.Value) {
673  bool Overflowed;
674  I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed);
675  if (Overflowed)
677  ++I;
678  continue;
679  }
680  ValueData.insert(I, J);
681  }
682 }
683 
685  function_ref<void(instrprof_error)> Warn) {
686  for (InstrProfValueData &I : ValueData) {
687  bool Overflowed;
688  I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
689  if (Overflowed)
691  }
692 }
693 
694 // Merge Value Profile data from Src record to this record for ValueKind.
695 // Scale merged value counts by \p Weight.
696 void InstrProfRecord::mergeValueProfData(
698  function_ref<void(instrprof_error)> Warn) {
699  uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
700  uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
701  if (ThisNumValueSites != OtherNumValueSites) {
703  return;
704  }
705  if (!ThisNumValueSites)
706  return;
707  std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
708  getOrCreateValueSitesForKind(ValueKind);
710  Src.getValueSitesForKind(ValueKind);
711  for (uint32_t I = 0; I < ThisNumValueSites; I++)
712  ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
713 }
714 
716  function_ref<void(instrprof_error)> Warn) {
717  // If the number of counters doesn't match we either have bad data
718  // or a hash collision.
719  if (Counts.size() != Other.Counts.size()) {
721  return;
722  }
723 
724  for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
725  bool Overflowed;
726  Counts[I] =
727  SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
728  if (Overflowed)
730  }
731 
732  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
733  mergeValueProfData(Kind, Other, Weight, Warn);
734 }
735 
736 void InstrProfRecord::scaleValueProfData(
738  function_ref<void(instrprof_error)> Warn) {
739  for (auto &R : getValueSitesForKind(ValueKind))
740  R.scale(N, D, Warn);
741 }
742 
744  function_ref<void(instrprof_error)> Warn) {
745  assert(D != 0 && "D cannot be 0");
746  for (auto &Count : this->Counts) {
747  bool Overflowed;
748  Count = SaturatingMultiply(Count, N, &Overflowed) / D;
749  if (Overflowed)
751  }
752  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
753  scaleValueProfData(Kind, N, D, Warn);
754 }
755 
756 // Map indirect call target name hash to name string.
757 uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
758  InstrProfSymtab *SymTab) {
759  if (!SymTab)
760  return Value;
761 
762  if (ValueKind == IPVK_IndirectCallTarget)
763  return SymTab->getFunctionHashFromAddress(Value);
764 
765  return Value;
766 }
767 
769  InstrProfValueData *VData, uint32_t N,
771  for (uint32_t I = 0; I < N; I++) {
772  VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
773  }
774  std::vector<InstrProfValueSiteRecord> &ValueSites =
775  getOrCreateValueSitesForKind(ValueKind);
776  if (N == 0)
777  ValueSites.emplace_back();
778  else
779  ValueSites.emplace_back(VData, VData + N);
780 }
781 
782 #define INSTR_PROF_COMMON_API_IMPL
784 
785 /*!
786  * ValueProfRecordClosure Interface implementation for InstrProfRecord
787  * class. These C wrappers are used as adaptors so that C++ code can be
788  * invoked as callbacks.
789  */
791  return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
792 }
793 
795  return reinterpret_cast<const InstrProfRecord *>(Record)
796  ->getNumValueSites(VKind);
797 }
798 
800  return reinterpret_cast<const InstrProfRecord *>(Record)
801  ->getNumValueData(VKind);
802 }
803 
805  uint32_t S) {
806  return reinterpret_cast<const InstrProfRecord *>(R)
807  ->getNumValueDataForSite(VK, S);
808 }
809 
810 void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
811  uint32_t K, uint32_t S) {
812  reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(Dst, K, S);
813 }
814 
815 ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
816  ValueProfData *VD =
817  (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
818  memset(VD, 0, TotalSizeInBytes);
819  return VD;
820 }
821 
822 static ValueProfRecordClosure InstrProfRecordClosure = {
823  nullptr,
828  nullptr,
831 
832 // Wrapper implementation using the closure mechanism.
833 uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
834  auto Closure = InstrProfRecordClosure;
835  Closure.Record = &Record;
836  return getValueProfDataSize(&Closure);
837 }
838 
839 // Wrapper implementation using the closure mechanism.
840 std::unique_ptr<ValueProfData>
841 ValueProfData::serializeFrom(const InstrProfRecord &Record) {
842  InstrProfRecordClosure.Record = &Record;
843 
844  std::unique_ptr<ValueProfData> VPD(
845  serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
846  return VPD;
847 }
848 
849 void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
850  InstrProfSymtab *SymTab) {
851  Record.reserveSites(Kind, NumValueSites);
852 
853  InstrProfValueData *ValueData = getValueProfRecordValueData(this);
854  for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
855  uint8_t ValueDataCount = this->SiteCountArray[VSite];
856  Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
857  ValueData += ValueDataCount;
858  }
859 }
860 
861 // For writing/serializing, Old is the host endianness, and New is
862 // byte order intended on disk. For Reading/deserialization, Old
863 // is the on-disk source endianness, and New is the host endianness.
864 void ValueProfRecord::swapBytes(support::endianness Old,
865  support::endianness New) {
866  using namespace support;
867 
868  if (Old == New)
869  return;
870 
871  if (getHostEndianness() != Old) {
872  sys::swapByteOrder<uint32_t>(NumValueSites);
873  sys::swapByteOrder<uint32_t>(Kind);
874  }
875  uint32_t ND = getValueProfRecordNumValueData(this);
876  InstrProfValueData *VD = getValueProfRecordValueData(this);
877 
878  // No need to swap byte array: SiteCountArrray.
879  for (uint32_t I = 0; I < ND; I++) {
880  sys::swapByteOrder<uint64_t>(VD[I].Value);
881  sys::swapByteOrder<uint64_t>(VD[I].Count);
882  }
883  if (getHostEndianness() == Old) {
884  sys::swapByteOrder<uint32_t>(NumValueSites);
885  sys::swapByteOrder<uint32_t>(Kind);
886  }
887 }
888 
889 void ValueProfData::deserializeTo(InstrProfRecord &Record,
890  InstrProfSymtab *SymTab) {
891  if (NumValueKinds == 0)
892  return;
893 
894  ValueProfRecord *VR = getFirstValueProfRecord(this);
895  for (uint32_t K = 0; K < NumValueKinds; K++) {
896  VR->deserializeTo(Record, SymTab);
897  VR = getValueProfRecordNext(VR);
898  }
899 }
900 
901 template <class T>
902 static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
903  using namespace support;
904 
905  if (Orig == little)
906  return endian::readNext<T, little, unaligned>(D);
907  else
908  return endian::readNext<T, big, unaligned>(D);
909 }
910 
911 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
912  return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
913  ValueProfData());
914 }
915 
916 Error ValueProfData::checkIntegrity() {
917  if (NumValueKinds > IPVK_Last + 1)
918  return make_error<InstrProfError>(
919  instrprof_error::malformed, "number of value profile kinds is invalid");
920  // Total size needs to be multiple of quadword size.
921  if (TotalSize % sizeof(uint64_t))
922  return make_error<InstrProfError>(
923  instrprof_error::malformed, "total size is not multiples of quardword");
924 
925  ValueProfRecord *VR = getFirstValueProfRecord(this);
926  for (uint32_t K = 0; K < this->NumValueKinds; K++) {
927  if (VR->Kind > IPVK_Last)
928  return make_error<InstrProfError>(instrprof_error::malformed,
929  "value kind is invalid");
930  VR = getValueProfRecordNext(VR);
931  if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
932  return make_error<InstrProfError>(
934  "value profile address is greater than total size");
935  }
936  return Error::success();
937 }
938 
940 ValueProfData::getValueProfData(const unsigned char *D,
941  const unsigned char *const BufferEnd,
943  using namespace support;
944 
945  if (D + sizeof(ValueProfData) > BufferEnd)
946  return make_error<InstrProfError>(instrprof_error::truncated);
947 
948  const unsigned char *Header = D;
949  uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
950  if (D + TotalSize > BufferEnd)
951  return make_error<InstrProfError>(instrprof_error::too_large);
952 
953  std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
954  memcpy(VPD.get(), D, TotalSize);
955  // Byte swap.
956  VPD->swapBytesToHost(Endianness);
957 
958  Error E = VPD->checkIntegrity();
959  if (E)
960  return std::move(E);
961 
962  return std::move(VPD);
963 }
964 
965 void ValueProfData::swapBytesToHost(support::endianness Endianness) {
966  using namespace support;
967 
968  if (Endianness == getHostEndianness())
969  return;
970 
971  sys::swapByteOrder<uint32_t>(TotalSize);
972  sys::swapByteOrder<uint32_t>(NumValueKinds);
973 
974  ValueProfRecord *VR = getFirstValueProfRecord(this);
975  for (uint32_t K = 0; K < NumValueKinds; K++) {
976  VR->swapBytes(Endianness, getHostEndianness());
977  VR = getValueProfRecordNext(VR);
978  }
979 }
980 
981 void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
982  using namespace support;
983 
984  if (Endianness == getHostEndianness())
985  return;
986 
987  ValueProfRecord *VR = getFirstValueProfRecord(this);
988  for (uint32_t K = 0; K < NumValueKinds; K++) {
989  ValueProfRecord *NVR = getValueProfRecordNext(VR);
990  VR->swapBytes(getHostEndianness(), Endianness);
991  VR = NVR;
992  }
993  sys::swapByteOrder<uint32_t>(TotalSize);
994  sys::swapByteOrder<uint32_t>(NumValueKinds);
995 }
996 
998  const InstrProfRecord &InstrProfR,
1000  uint32_t MaxMDCount) {
1001  uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx);
1002  if (!NV)
1003  return;
1004 
1005  uint64_t Sum = 0;
1006  std::unique_ptr<InstrProfValueData[]> VD =
1007  InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum);
1008 
1009  ArrayRef<InstrProfValueData> VDs(VD.get(), NV);
1010  annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1011 }
1012 
1016  uint32_t MaxMDCount) {
1017  LLVMContext &Ctx = M.getContext();
1018  MDBuilder MDHelper(Ctx);
1020  // Tag
1021  Vals.push_back(MDHelper.createString("VP"));
1022  // Value Kind
1023  Vals.push_back(MDHelper.createConstant(
1025  // Total Count
1026  Vals.push_back(
1027  MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
1028 
1029  // Value Profile Data
1030  uint32_t MDCount = MaxMDCount;
1031  for (auto &VD : VDs) {
1032  Vals.push_back(MDHelper.createConstant(
1033  ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
1034  Vals.push_back(MDHelper.createConstant(
1035  ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
1036  if (--MDCount == 0)
1037  break;
1038  }
1039  Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
1040 }
1041 
1044  uint32_t MaxNumValueData,
1045  InstrProfValueData ValueData[],
1046  uint32_t &ActualNumValueData, uint64_t &TotalC,
1047  bool GetNoICPValue) {
1048  MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1049  if (!MD)
1050  return false;
1051 
1052  unsigned NOps = MD->getNumOperands();
1053 
1054  if (NOps < 5)
1055  return false;
1056 
1057  // Operand 0 is a string tag "VP":
1058  MDString *Tag = cast<MDString>(MD->getOperand(0));
1059  if (!Tag)
1060  return false;
1061 
1062  if (!Tag->getString().equals("VP"))
1063  return false;
1064 
1065  // Now check kind:
1066  ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1067  if (!KindInt)
1068  return false;
1069  if (KindInt->getZExtValue() != ValueKind)
1070  return false;
1071 
1072  // Get total count
1073  ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1074  if (!TotalCInt)
1075  return false;
1076  TotalC = TotalCInt->getZExtValue();
1077 
1078  ActualNumValueData = 0;
1079 
1080  for (unsigned I = 3; I < NOps; I += 2) {
1081  if (ActualNumValueData >= MaxNumValueData)
1082  break;
1083  ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1084  ConstantInt *Count =
1085  mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1086  if (!Value || !Count)
1087  return false;
1088  uint64_t CntValue = Count->getZExtValue();
1089  if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1090  continue;
1091  ValueData[ActualNumValueData].Value = Value->getZExtValue();
1092  ValueData[ActualNumValueData].Count = CntValue;
1093  ActualNumValueData++;
1094  }
1095  return true;
1096 }
1097 
1099  return F.getMetadata(getPGOFuncNameMetadataName());
1100 }
1101 
1103  // Only for internal linkage functions.
1104  if (PGOFuncName == F.getName())
1105  return;
1106  // Don't create duplicated meta-data.
1108  return;
1109  LLVMContext &C = F.getContext();
1110  MDNode *N = MDNode::get(C, MDString::get(C, PGOFuncName));
1111  F.setMetadata(getPGOFuncNameMetadataName(), N);
1112 }
1113 
1114 bool needsComdatForCounter(const Function &F, const Module &M) {
1115  if (F.hasComdat())
1116  return true;
1117 
1118  if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1119  return false;
1120 
1121  // See createPGOFuncNameVar for more details. To avoid link errors, profile
1122  // counters for function with available_externally linkage needs to be changed
1123  // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1124  // created. Without using comdat, duplicate entries won't be removed by the
1125  // linker leading to increased data segement size and raw profile size. Even
1126  // worse, since the referenced counter from profile per-function data object
1127  // will be resolved to the common strong definition, the profile counts for
1128  // available_externally functions will end up being duplicated in raw profile
1129  // data. This can result in distorted profile as the counts of those dups
1130  // will be accumulated by the profile merger.
1131  GlobalValue::LinkageTypes Linkage = F.getLinkage();
1134  return false;
1135 
1136  return true;
1137 }
1138 
1139 // Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1140 bool isIRPGOFlagSet(const Module *M) {
1141  auto IRInstrVar =
1142  M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1143  if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1144  return false;
1145 
1146  // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1147  // have the decl.
1148  if (IRInstrVar->isDeclaration())
1149  return true;
1150 
1151  // Check if the flag is set.
1152  if (!IRInstrVar->hasInitializer())
1153  return false;
1154 
1155  auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1156  if (!InitVal)
1157  return false;
1158  return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1159 }
1160 
1161 // Check if we can safely rename this Comdat function.
1162 bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1163  if (F.getName().empty())
1164  return false;
1165  if (!needsComdatForCounter(F, *(F.getParent())))
1166  return false;
1167  // Unsafe to rename the address-taken function (which can be used in
1168  // function comparison).
1169  if (CheckAddressTaken && F.hasAddressTaken())
1170  return false;
1171  // Only safe to do if this function may be discarded if it is not used
1172  // in the compilation unit.
1173  if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1174  return false;
1175 
1176  // For AvailableExternallyLinkage functions.
1177  if (!F.hasComdat()) {
1179  return true;
1180  }
1181  return true;
1182 }
1183 
1184 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1185 // aware this is an ir_level profile so it can set the version flag.
1187  bool InstrEntryBBEnabled,
1188  bool DebugInfoCorrelate,
1189  bool PGOFunctionEntryCoverage) {
1190  const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1191  Type *IntTy64 = Type::getInt64Ty(M.getContext());
1192  uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
1193  if (IsCS)
1194  ProfileVersion |= VARIANT_MASK_CSIR_PROF;
1195  if (InstrEntryBBEnabled)
1196  ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
1197  if (DebugInfoCorrelate)
1198  ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
1200  ProfileVersion |=
1201  VARIANT_MASK_BYTE_COVERAGE | VARIANT_MASK_FUNCTION_ENTRY_ONLY;
1202  auto IRLevelVersionVariable = new GlobalVariable(
1203  M, IntTy64, true, GlobalValue::WeakAnyLinkage,
1204  Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
1205  IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
1206  Triple TT(M.getTargetTriple());
1207  if (TT.supportsCOMDAT()) {
1208  IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage);
1209  IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
1210  }
1211  return IRLevelVersionVariable;
1212 }
1213 
1214 // Create the variable for the profile file name.
1215 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1216  if (InstrProfileOutput.empty())
1217  return;
1218  Constant *ProfileNameConst =
1219  ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1220  GlobalVariable *ProfileNameVar = new GlobalVariable(
1221  M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1222  ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1223  Triple TT(M.getTargetTriple());
1224  if (TT.supportsCOMDAT()) {
1225  ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
1226  ProfileNameVar->setComdat(M.getOrInsertComdat(
1227  StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1228  }
1229 }
1230 
1231 Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1232  const std::string &TestFilename,
1233  bool IsCS) {
1234  auto getProfileSum = [IsCS](const std::string &Filename,
1235  CountSumOrPercent &Sum) -> Error {
1236  auto ReaderOrErr = InstrProfReader::create(Filename);
1237  if (Error E = ReaderOrErr.takeError()) {
1238  return E;
1239  }
1240  auto Reader = std::move(ReaderOrErr.get());
1241  Reader->accumulateCounts(Sum, IsCS);
1242  return Error::success();
1243  };
1244  auto Ret = getProfileSum(BaseFilename, Base);
1245  if (Ret)
1246  return Ret;
1247  Ret = getProfileSum(TestFilename, Test);
1248  if (Ret)
1249  return Ret;
1250  this->BaseFilename = &BaseFilename;
1251  this->TestFilename = &TestFilename;
1252  Valid = true;
1253  return Error::success();
1254 }
1255 
1257  Mismatch.NumEntries += 1;
1258  Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1259  for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1260  if (Test.ValueCounts[I] >= 1.0f)
1261  Mismatch.ValueCounts[I] +=
1262  MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1263  }
1264 }
1265 
1267  Unique.NumEntries += 1;
1268  Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1269  for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1270  if (Test.ValueCounts[I] >= 1.0f)
1271  Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1272  }
1273 }
1274 
1276  if (!Valid)
1277  return;
1278 
1279  const char *EntryName =
1280  (Level == ProgramLevel ? "functions" : "edge counters");
1281  if (Level == ProgramLevel) {
1282  OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1283  << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1284  } else {
1285  OS << "Function level:\n"
1286  << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1287  }
1288 
1289  OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1290  if (Mismatch.NumEntries)
1291  OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1292  << "\n";
1293  if (Unique.NumEntries)
1294  OS << " # of " << EntryName
1295  << " only in test_profile: " << Unique.NumEntries << "\n";
1296 
1297  OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1298  << "\n";
1299  if (Mismatch.NumEntries)
1300  OS << " Mismatched count percentage (Edge): "
1301  << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1302  if (Unique.NumEntries)
1303  OS << " Percentage of Edge profile only in test_profile: "
1304  << format("%.3f%%", Unique.CountSum * 100) << "\n";
1305  OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)
1306  << "\n"
1307  << " Edge profile test count sum: " << format("%.0f", Test.CountSum)
1308  << "\n";
1309 
1310  for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1311  if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1312  continue;
1313  char ProfileKindName[20];
1314  switch (I) {
1315  case IPVK_IndirectCallTarget:
1316  strncpy(ProfileKindName, "IndirectCall", 19);
1317  break;
1318  case IPVK_MemOPSize:
1319  strncpy(ProfileKindName, "MemOP", 19);
1320  break;
1321  default:
1322  snprintf(ProfileKindName, 19, "VP[%d]", I);
1323  break;
1324  }
1325  OS << " " << ProfileKindName
1326  << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1327  << "\n";
1328  if (Mismatch.NumEntries)
1329  OS << " Mismatched count percentage (" << ProfileKindName
1330  << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1331  if (Unique.NumEntries)
1332  OS << " Percentage of " << ProfileKindName
1333  << " profile only in test_profile: "
1334  << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1335  OS << " " << ProfileKindName
1336  << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1337  << "\n"
1338  << " " << ProfileKindName
1339  << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1340  << "\n";
1341  }
1342 }
1343 
1344 } // end namespace llvm
llvm::SaturatingMultiply
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:820
llvm::OverlapStats::Unique
CountSumOrPercent Unique
Definition: InstrProf.h:646
llvm::OverlapStats::Level
OverlapStatsLevel Level
Definition: InstrProf.h:647
llvm::createPGOFuncNameMetadata
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1102
Compression.h
llvm::OverlapStats::score
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:670
llvm::allocValueProfData
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
Definition: InstrProf.cpp:911
llvm::StringRef::startswith
LLVM_NODISCARD bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:286
llvm::instrprof_error::bad_header
@ bad_header
MathExtras.h
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: Globals.cpp:192
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:22
llvm::MDBuilder::createConstant
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
Definition: MDBuilder.cpp:24
llvm::msgpack::Endianness
constexpr support::endianness Endianness
The endianness of all multi-byte encoded values in MessagePack.
Definition: MsgPack.h:24
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::GlobalValue::getGlobalIdentifier
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:164
llvm::GlobalValue::LinkOnceAnyLinkage
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
Definition: GlobalValue.h:50
Metadata.h
llvm::OverlapStats::Valid
bool Valid
Definition: InstrProf.h:652
llvm::InstrProfError::message
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:257
llvm::getNumValueKindsInstrProf
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
Definition: InstrProf.cpp:790
llvm::instrprof_error::uncompress_failed
@ uncompress_failed
llvm::GlobalValue::HiddenVisibility
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
T
PGOFunctionEntryCoverage
static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Use this option to enable function entry coverage instrumentation."))
llvm::Function
Definition: Function.h:62
StringRef.h
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:632
llvm::InstrProfRecord::getValueForSite
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
Definition: InstrProf.h:918
llvm::needsComdatForCounter
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1114
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:480
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1176
Path.h
llvm::GlobalValue::isLocalLinkage
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:332
llvm::getPGOFuncName
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Definition: InstrProf.cpp:301
ErrorHandling.h
llvm::zlib::uncompress
Error uncompress(StringRef InputBuffer, char *UncompressedBuffer, size_t &UncompressedSize)
Definition: Compression.cpp:63
llvm::InstrProfSymtab::create
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
llvm::OverlapStats::dump
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1275
ManagedStatic.h
llvm::Error::success
static ErrorSuccess success()
Create a success value.
Definition: Error.h:329
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::getInstrProfNameSeparator
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:164
llvm::getValueForSiteInstrProf
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
Definition: InstrProf.cpp:810
Error.h
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
SwapByteOrder.h
llvm::createPGOFuncNameVar
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:380
llvm::tgtok::VarName
@ VarName
Definition: TGLexer.h:71
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
StaticFuncFullModulePrefix
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
llvm::instrprof_error::zlib_unavailable
@ zlib_unavailable
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::OverlapStats
Definition: InstrProf.h:637
Module.h
llvm::InstrProfValueSiteRecord::scale
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:684
llvm::InstrProfRecord::overlap
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:611
llvm::instrprof_error::value_site_count_mismatch
@ value_site_count_mismatch
llvm::GlobalValue::DefaultVisibility
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:116
llvm::Expected
Tagged union holding either a T or a Error.
Definition: APFloat.h:42
llvm::GlobalValue::LinkageTypes
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
llvm::getPGOFuncNameVarName
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:335
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1215
llvm::instrprof_error::invalid_prof
@ invalid_prof
LLVM_ATTRIBUTE_UNUSED
#define LLVM_ATTRIBUTE_UNUSED
Definition: Compiler.h:189
llvm::GlobalValue::isDiscardableIfUnused
bool isDiscardableIfUnused() const
Definition: GlobalValue.h:469
llvm::consumeError
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1043
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:241
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:158
llvm::InstrProfRecord
Profiling information for a single function.
Definition: InstrProf.h:717
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1233
F
#define F(x, y, z)
Definition: MD5.cpp:55
InstrProfData.inc
llvm::Instruction::setMetadata
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1336
llvm::InstrProfValueSiteRecord::merge
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:662
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1143
llvm::instrprof_error::unexpected_debug_info_for_correlation
@ unexpected_debug_info_for_correlation
llvm::OverlapStats::TestFilename
const std::string * TestFilename
Definition: InstrProf.h:649
getInstrProfErrString
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Definition: InstrProf.cpp:77
Instruction.h
CommandLine.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:73
llvm::decodeULEB128
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
Definition: LEB128.h:128
llvm::createIRLevelProfileFlagVar
GlobalVariable * createIRLevelProfileFlagVar(Module &M, bool IsCS, bool InstrEntryBBEnabled, bool DebugInfoCorrelate, bool PGOFunctionEntryCoverage)
Definition: InstrProf.cpp:1186
GlobalValue.h
llvm::InstrProfRecord::getNumValueDataForSite
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:912
llvm::MutableArrayRef
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:305
ptrdiff_t
Constants.h
llvm::CountSumOrPercent::CountSum
double CountSum
Definition: InstrProf.h:625
SmallString.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::zlib::BestSizeCompression
static constexpr int BestSizeCompression
Definition: Compression.h:28
llvm::support::little
@ little
Definition: Endian.h:27
llvm::StringRef::split
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:749
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::InstrProfRecordClosure
static ValueProfRecordClosure InstrProfRecordClosure
Definition: InstrProf.cpp:822
llvm::canRenameComdatFunc
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1162
llvm::ManagedStatic
ManagedStatic - This transparently changes the behavior of global statics to be lazily constructed on...
Definition: ManagedStatic.h:83
llvm::instrprof_error::compress_failed
@ compress_failed
llvm::InstrProfRecord::getNumValueSites
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:908
llvm::allocValueProfDataInstrProf
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
Definition: InstrProf.cpp:815
llvm::Triple::supportsCOMDAT
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:816
llvm::stripDirPrefix
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
Definition: InstrProf.cpp:273
llvm::InstrProfError::ID
static char ID
Definition: InstrProf.h:354
llvm::Instruction
Definition: Instruction.h:45
InstrProf.h
llvm::swapToHostOrder
static T swapToHostOrder(const unsigned char *&D, support::endianness Orig)
Definition: InstrProf.cpp:902
llvm::instrprof_error::truncated
@ truncated
MDBuilder.h
llvm::zlib::compress
Error compress(StringRef InputBuffer, SmallVectorImpl< char > &CompressedBuffer, int Level=DefaultCompression)
Definition: Compression.cpp:49
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:925
llvm::OverlapStats::FuncName
StringRef FuncName
Definition: InstrProf.h:650
StaticFuncStripDirNamePrefix
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
llvm::instrprof_error::bad_magic
@ bad_magic
llvm::instrprof_error::unsupported_version
@ unsupported_version
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
Type.h
llvm::instrprof_error::too_large
@ too_large
llvm::InstrProfSectKind
InstrProfSectKind
Definition: InstrProf.h:58
llvm::Instruction::getMetadata
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:282
llvm::SmallString< 128 >
llvm::ARM_PROC::IE
@ IE
Definition: ARMBaseInfo.h:27
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1137
llvm::MDBuilder::createString
MDString * createString(StringRef Str)
Return the given string as metadata.
Definition: MDBuilder.cpp:20
llvm::getHostEndianness
support::endianness getHostEndianness()
Definition: InstrProf.h:954
llvm::getInstrProfNameVarPrefix
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:85
llvm::instrprof_error::unsupported_hash_type
@ unsupported_hash_type
llvm::cl::opt< bool >
InstrProfReader.h
llvm::CountSumOrPercent::NumEntries
uint64_t NumEntries
Definition: InstrProf.h:624
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:135
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::StringRef::empty
constexpr LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:153
llvm::count
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1675
uint64_t
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::GlobalValue::WeakAnyLinkage
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
llvm::SoftInstrProfErrors::addError
void addError(instrprof_error IE)
Track a soft error (IE) and increment its associated counter.
Definition: InstrProf.cpp:232
llvm::getNumValueSitesInstrProf
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:794
LEB128.h
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::CountSumOrPercent
Definition: InstrProf.h:623
llvm::isIRPGOFlagSet
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1140
llvm::instrprof_error::empty_raw_profile
@ empty_raw_profile
I
#define I(x, y, z)
Definition: MD5.cpp:58
StringExtras.h
llvm::instrprof_error
instrprof_error
Definition: InstrProf.h:295
llvm::instrprof_error::success
@ success
llvm::SaturatingMultiplyAdd
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition: MathExtras.h:866
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:441
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:460
ArrayRef.h
llvm::HighlightColor::Address
@ Address
llvm::MDString::get
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:473
message
message(STATUS "Targeting ${t}") add_subdirectory($
Definition: CMakeLists.txt:33
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::instrprof_error::unknown_function
@ unknown_function
memcpy
<%struct.s * > cast struct s *S to sbyte *< sbyte * > sbyte uint cast struct s *agg result to sbyte *< sbyte * > sbyte uint cast struct s *memtmp to sbyte *< sbyte * > sbyte uint ret void llc ends up issuing two memcpy or custom lower memcpy(of small size) to be ldmia/stmia. I think option 2 is better but the current register allocator cannot allocate a chunk of registers at a time. A feasible temporary solution is to use specific physical registers at the lowering time for small(<
llvm::instrprof_error::unrecognized_format
@ unrecognized_format
llvm::zlib::isAvailable
bool isAvailable()
Definition: Compression.cpp:47
llvm::Record
Definition: Record.h:1485
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::InstrProfValueKind
InstrProfValueKind
Definition: InstrProf.h:239
llvm::MDNode
Metadata node.
Definition: Metadata.h:906
llvm::collectPGOFuncNameStrings
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:439
llvm::getNumValueDataInstrProf
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
Definition: InstrProf.cpp:799
llvm::StringRef::bytes_begin
const unsigned char * bytes_begin() const
Definition: StringRef.h:132
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::InstrProfSymtab::getFunctionHashFromAddress
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:425
Triple.h
llvm::InstrProfRecord::accumulateCounts
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:542
llvm::OverlapStats::addOneUnique
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1266
llvm::ArrayRef< std::string >
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
ErrorCategory
static ManagedStatic< InstrProfErrorCategoryType > ErrorCategory
Definition: InstrProf.cpp:179
uint32_t
llvm::instrprof_error::eof
@ eof
Compiler.h
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:3086
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::format
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:124
llvm::GlobalValue::AvailableExternallyLinkage
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
llvm::instrprof_error::hash_mismatch
@ hash_mismatch
llvm::Triple::ObjectFormatType
ObjectFormatType
Definition: Triple.h:237
llvm::ValueMap
See the file comment.
Definition: ValueMap.h:85
llvm::InstrProfValueSiteRecord::ValueData
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:687
llvm::raw_fd_ostream
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:444
llvm::instrprof_category
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:181
llvm::annotateValueSite
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:997
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
llvm::InstrProfRecord::merge
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:715
llvm::DoInstrProfNameCompression
cl::opt< bool > DoInstrProfNameCompression
llvm::InstrProfRecord::overlapValueProfData
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:593
llvm::readPGOFuncNameStrings
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
Definition: InstrProf.cpp:497
llvm::StringRef::size
constexpr LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:157
name
static const char * name
Definition: SVEIntrinsicOpts.cpp:74
llvm::partition_point
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
Definition: STLExtras.h:1740
Constant.h
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:242
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::InstrProfSymtab
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:433
llvm::StringRef::drop_front
LLVM_NODISCARD StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition: StringRef.h:653
llvm::InstrProfValueSiteRecord::overlap
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:562
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:151
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:155
GlobalVariable.h
llvm::GlobalValue::getGUID
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:517
Casting.h
llvm::Triple::MachO
@ MachO
Definition: Triple.h:243
Function.h
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::OverlapStats::ProgramLevel
@ ProgramLevel
Definition: InstrProf.h:638
llvm::InstrProfReader::create
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, const InstrProfCorrelator *Correlator=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
Definition: InstrProfReader.cpp:55
llvm::instrprof_error::count_mismatch
@ count_mismatch
llvm::NOMORE_ICP_MAGICNUM
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
Definition: Metadata.h:57
llvm::OverlapStats::Overlap
CountSumOrPercent Overlap
Definition: InstrProf.h:644
llvm::getNumValueDataForSiteInstrProf
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
Definition: InstrProf.cpp:804
llvm::OverlapStats::Mismatch
CountSumOrPercent Mismatch
Definition: InstrProf.h:645
llvm::getPGOFuncNameMetadataName
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:267
llvm::InstrProfRecord::addValueData
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:768
llvm::MDBuilder
Definition: MDBuilder.h:35
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:213
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
llvm::sys::path::is_separator
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
Definition: Path.cpp:602
llvm::InstrProfValueSiteRecord
Definition: InstrProf.h:685
llvm::InstrProfSymtab::addFuncName
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:494
llvm::Triple::COFF
@ COFF
Definition: Triple.h:240
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
llvm::Constant::getIntegerValue
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
Definition: Constants.cpp:388
Version
uint64_t Version
Definition: RawMemProfReader.cpp:25
SmallVector.h
llvm::OverlapStats::FuncHash
uint64_t FuncHash
Definition: InstrProf.h:651
N
#define N
llvm::OverlapStats::BaseFilename
const std::string * BaseFilename
Definition: InstrProf.h:648
llvm::instrprof_error::unable_to_correlate_profile
@ unable_to_correlate_profile
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
support
Reimplement select in terms of SEL *We would really like to support but we need to prove that the add doesn t need to overflow between the two bit chunks *Implement pre post increment support(e.g. PR935) *Implement smarter const ant generation for binops with large immediates. A few ARMv6T2 ops should be pattern matched
Definition: README.txt:10
llvm::encodeULEB128
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
llvm::support::endianness
endianness
Definition: Endian.h:27
llvm::OverlapStats::Base
CountSumOrPercent Base
Definition: InstrProf.h:640
llvm::InstrProfValueSiteRecord::sortByTargetValues
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:695
llvm::instrprof_error::malformed
@ malformed
llvm::CountSumOrPercent::ValueCounts
double ValueCounts[IPVK_Last - IPVK_First+1]
Definition: InstrProf.h:626
llvm::getFuncNameWithoutPrefix
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:324
llvm::OverlapStats::accumulateCounts
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1231
llvm::getValueProfDataFromInst
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1042
LLVMContext.h
llvm::cl::desc
Definition: CommandLine.h:412
llvm::GlobalValue::ExternalWeakLinkage
@ ExternalWeakLinkage
ExternalWeak linkage description.
Definition: GlobalValue.h:57
error_category
static ManagedStatic< _object_error_category > error_category
Definition: Error.cpp:76
Endian.h
llvm::MDString
A single uniqued string.
Definition: Metadata.h:611
llvm::GlobalValue::LinkOnceODRLinkage
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
llvm::raw_string_ostream::str
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:650
llvm::DebugInfoCorrelate
cl::opt< bool > DebugInfoCorrelate("debug-info-correlate", cl::ZeroOrMore, cl::desc("Use debug info to correlate profiles."), cl::init(false))
Definition: PGOInstrumentation.cpp:300
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::OverlapStats::addOneMismatch
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1256
llvm::InstrProfRecord::Counts
std::vector< uint64_t > Counts
Definition: InstrProf.h:718
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:152
llvm::instrprof_error::counter_overflow
@ counter_overflow
llvm::OverlapStats::Test
CountSumOrPercent Test
Definition: InstrProf.h:642
llvm::instrprof_error::missing_debug_info_for_correlation
@ missing_debug_info_for_correlation
Other
Optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1198
llvm::getPGOFuncNameMetadata
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1098
llvm::StringRef::bytes_end
const unsigned char * bytes_end() const
Definition: StringRef.h:135
llvm::InstrProfRecord::scale
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:743