LLVM  4.0.0
SampleProfile.cpp
Go to the documentation of this file.
1 //===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the SampleProfileLoader transformation. This pass
11 // reads a profile file generated by a sampling profiler (e.g. Linux Perf -
12 // http://perf.wiki.kernel.org/) and generates IR metadata to reflect the
13 // profile information in the given profile.
14 //
15 // This pass generates branch weight annotations on the IR:
16 //
17 // - prof: Represents branch weights. This annotation is added to branches
18 // to indicate the weights of each edge coming out of the branch.
19 // The weight of each edge is the weight of the target block for
20 // that edge. The weight of a block B is computed as the maximum
21 // number of samples found in B.
22 //
23 //===----------------------------------------------------------------------===//
24 
26 #include "llvm/ADT/DenseMap.h"
27 #include "llvm/ADT/SmallPtrSet.h"
28 #include "llvm/ADT/SmallSet.h"
29 #include "llvm/ADT/StringRef.h"
31 #include "llvm/Analysis/LoopInfo.h"
33 #include "llvm/IR/Constants.h"
34 #include "llvm/IR/DebugInfo.h"
35 #include "llvm/IR/DiagnosticInfo.h"
36 #include "llvm/IR/Dominators.h"
37 #include "llvm/IR/Function.h"
38 #include "llvm/IR/InstIterator.h"
39 #include "llvm/IR/Instructions.h"
40 #include "llvm/IR/IntrinsicInst.h"
41 #include "llvm/IR/LLVMContext.h"
42 #include "llvm/IR/MDBuilder.h"
43 #include "llvm/IR/Metadata.h"
44 #include "llvm/IR/Module.h"
45 #include "llvm/Pass.h"
48 #include "llvm/Support/Debug.h"
49 #include "llvm/Support/ErrorOr.h"
50 #include "llvm/Support/Format.h"
52 #include "llvm/Transforms/IPO.h"
54 #include <cctype>
55 
56 using namespace llvm;
57 using namespace sampleprof;
58 
59 #define DEBUG_TYPE "sample-profile"
60 
61 // Command line option to specify the file to read samples from. This is
62 // mainly used for debugging.
64  "sample-profile-file", cl::init(""), cl::value_desc("filename"),
65  cl::desc("Profile file loaded by -sample-profile"), cl::Hidden);
67  "sample-profile-max-propagate-iterations", cl::init(100),
68  cl::desc("Maximum number of iterations to go through when propagating "
69  "sample block/edge weights through the CFG."));
71  "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
72  cl::desc("Emit a warning if less than N% of records in the input profile "
73  "are matched to the IR."));
75  "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
76  cl::desc("Emit a warning if less than N% of samples in the input profile "
77  "are matched to the IR."));
79  "sample-profile-inline-hot-threshold", cl::init(0.1), cl::value_desc("N"),
80  cl::desc("Inlined functions that account for more than N% of all samples "
81  "collected in the parent function, will be inlined again."));
82 
83 namespace {
84 typedef DenseMap<const BasicBlock *, uint64_t> BlockWeightMap;
85 typedef DenseMap<const BasicBlock *, const BasicBlock *> EquivalenceClassMap;
86 typedef std::pair<const BasicBlock *, const BasicBlock *> Edge;
87 typedef DenseMap<Edge, uint64_t> EdgeWeightMap;
89  BlockEdgeMap;
90 
91 class SampleCoverageTracker {
92 public:
93  SampleCoverageTracker() : SampleCoverage(), TotalUsedSamples(0) {}
94 
95  bool markSamplesUsed(const FunctionSamples *FS, uint32_t LineOffset,
96  uint32_t Discriminator, uint64_t Samples);
97  unsigned computeCoverage(unsigned Used, unsigned Total) const;
98  unsigned countUsedRecords(const FunctionSamples *FS) const;
99  unsigned countBodyRecords(const FunctionSamples *FS) const;
100  uint64_t getTotalUsedSamples() const { return TotalUsedSamples; }
101  uint64_t countBodySamples(const FunctionSamples *FS) const;
102  void clear() {
103  SampleCoverage.clear();
104  TotalUsedSamples = 0;
105  }
106 
107 private:
108  typedef std::map<LineLocation, unsigned> BodySampleCoverageMap;
110  FunctionSamplesCoverageMap;
111 
112  /// Coverage map for sampling records.
113  ///
114  /// This map keeps a record of sampling records that have been matched to
115  /// an IR instruction. This is used to detect some form of staleness in
116  /// profiles (see flag -sample-profile-check-coverage).
117  ///
118  /// Each entry in the map corresponds to a FunctionSamples instance. This is
119  /// another map that counts how many times the sample record at the
120  /// given location has been used.
121  FunctionSamplesCoverageMap SampleCoverage;
122 
123  /// Number of samples used from the profile.
124  ///
125  /// When a sampling record is used for the first time, the samples from
126  /// that record are added to this accumulator. Coverage is later computed
127  /// based on the total number of samples available in this function and
128  /// its callsites.
129  ///
130  /// Note that this accumulator tracks samples used from a single function
131  /// and all the inlined callsites. Strictly, we should have a map of counters
132  /// keyed by FunctionSamples pointers, but these stats are cleared after
133  /// every function, so we just need to keep a single counter.
134  uint64_t TotalUsedSamples;
135 };
136 
137 /// \brief Sample profile pass.
138 ///
139 /// This pass reads profile data from the file specified by
140 /// -sample-profile-file and annotates every affected function with the
141 /// profile information found in that file.
142 class SampleProfileLoader {
143 public:
144  SampleProfileLoader(StringRef Name = SampleProfileFile)
145  : DT(nullptr), PDT(nullptr), LI(nullptr), ACT(nullptr), Reader(),
146  Samples(nullptr), Filename(Name), ProfileIsValid(false),
147  TotalCollectedSamples(0) {}
148 
149  bool doInitialization(Module &M);
150  bool runOnModule(Module &M);
151  void setACT(AssumptionCacheTracker *A) { ACT = A; }
152 
153  void dump() { Reader->dump(); }
154 
155 protected:
156  bool runOnFunction(Function &F);
157  unsigned getFunctionLoc(Function &F);
158  bool emitAnnotations(Function &F);
159  ErrorOr<uint64_t> getInstWeight(const Instruction &I);
160  ErrorOr<uint64_t> getBlockWeight(const BasicBlock *BB);
161  const FunctionSamples *findCalleeFunctionSamples(const Instruction &I) const;
162  const FunctionSamples *findFunctionSamples(const Instruction &I) const;
163  bool inlineHotFunctions(Function &F);
164  void printEdgeWeight(raw_ostream &OS, Edge E);
165  void printBlockWeight(raw_ostream &OS, const BasicBlock *BB) const;
166  void printBlockEquivalence(raw_ostream &OS, const BasicBlock *BB);
167  bool computeBlockWeights(Function &F);
168  void findEquivalenceClasses(Function &F);
169  void findEquivalencesFor(BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
171  void propagateWeights(Function &F);
172  uint64_t visitEdge(Edge E, unsigned *NumUnknownEdges, Edge *UnknownEdge);
173  void buildEdges(Function &F);
174  bool propagateThroughEdges(Function &F, bool UpdateBlockCount);
175  void computeDominanceAndLoopInfo(Function &F);
176  unsigned getOffset(unsigned L, unsigned H) const;
177  void clearFunctionData();
178 
179  /// \brief Map basic blocks to their computed weights.
180  ///
181  /// The weight of a basic block is defined to be the maximum
182  /// of all the instruction weights in that block.
183  BlockWeightMap BlockWeights;
184 
185  /// \brief Map edges to their computed weights.
186  ///
187  /// Edge weights are computed by propagating basic block weights in
188  /// SampleProfile::propagateWeights.
189  EdgeWeightMap EdgeWeights;
190 
191  /// \brief Set of visited blocks during propagation.
193 
194  /// \brief Set of visited edges during propagation.
195  SmallSet<Edge, 32> VisitedEdges;
196 
197  /// \brief Equivalence classes for block weights.
198  ///
199  /// Two blocks BB1 and BB2 are in the same equivalence class if they
200  /// dominate and post-dominate each other, and they are in the same loop
201  /// nest. When this happens, the two blocks are guaranteed to execute
202  /// the same number of times.
203  EquivalenceClassMap EquivalenceClass;
204 
205  /// \brief Dominance, post-dominance and loop information.
206  std::unique_ptr<DominatorTree> DT;
207  std::unique_ptr<DominatorTreeBase<BasicBlock>> PDT;
208  std::unique_ptr<LoopInfo> LI;
209 
211 
212  /// \brief Predecessors for each basic block in the CFG.
213  BlockEdgeMap Predecessors;
214 
215  /// \brief Successors for each basic block in the CFG.
216  BlockEdgeMap Successors;
217 
218  SampleCoverageTracker CoverageTracker;
219 
220  /// \brief Profile reader object.
221  std::unique_ptr<SampleProfileReader> Reader;
222 
223  /// \brief Samples collected for the body of this function.
224  FunctionSamples *Samples;
225 
226  /// \brief Name of the profile file to load.
227  std::string Filename;
228 
229  /// \brief Flag indicating whether the profile input loaded successfully.
230  bool ProfileIsValid;
231 
232  /// \brief Total number of samples collected in this profile.
233  ///
234  /// This is the sum of all the samples collected in all the functions executed
235  /// at runtime.
236  uint64_t TotalCollectedSamples;
237 };
238 
239 class SampleProfileLoaderLegacyPass : public ModulePass {
240 public:
241  // Class identification, replacement for typeinfo
242  static char ID;
243 
244  SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile)
245  : ModulePass(ID), SampleLoader(Name) {
248  }
249 
250  void dump() { SampleLoader.dump(); }
251 
252  bool doInitialization(Module &M) override {
253  return SampleLoader.doInitialization(M);
254  }
255  StringRef getPassName() const override { return "Sample profile pass"; }
256  bool runOnModule(Module &M) override;
257 
258  void getAnalysisUsage(AnalysisUsage &AU) const override {
260  }
261 
262 private:
263  SampleProfileLoader SampleLoader;
264 };
265 
266 /// Return true if the given callsite is hot wrt to its caller.
267 ///
268 /// Functions that were inlined in the original binary will be represented
269 /// in the inline stack in the sample profile. If the profile shows that
270 /// the original inline decision was "good" (i.e., the callsite is executed
271 /// frequently), then we will recreate the inline decision and apply the
272 /// profile from the inlined callsite.
273 ///
274 /// To decide whether an inlined callsite is hot, we compute the fraction
275 /// of samples used by the callsite with respect to the total number of samples
276 /// collected in the caller.
277 ///
278 /// If that fraction is larger than the default given by
279 /// SampleProfileHotThreshold, the callsite will be inlined again.
280 bool callsiteIsHot(const FunctionSamples *CallerFS,
281  const FunctionSamples *CallsiteFS) {
282  if (!CallsiteFS)
283  return false; // The callsite was not inlined in the original binary.
284 
285  uint64_t ParentTotalSamples = CallerFS->getTotalSamples();
286  if (ParentTotalSamples == 0)
287  return false; // Avoid division by zero.
288 
289  uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
290  if (CallsiteTotalSamples == 0)
291  return false; // Callsite is trivially cold.
292 
293  double PercentSamples =
294  (double)CallsiteTotalSamples / (double)ParentTotalSamples * 100.0;
295  return PercentSamples >= SampleProfileHotThreshold;
296 }
297 }
298 
299 /// Mark as used the sample record for the given function samples at
300 /// (LineOffset, Discriminator).
301 ///
302 /// \returns true if this is the first time we mark the given record.
303 bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
304  uint32_t LineOffset,
305  uint32_t Discriminator,
306  uint64_t Samples) {
307  LineLocation Loc(LineOffset, Discriminator);
308  unsigned &Count = SampleCoverage[FS][Loc];
309  bool FirstTime = (++Count == 1);
310  if (FirstTime)
311  TotalUsedSamples += Samples;
312  return FirstTime;
313 }
314 
315 /// Return the number of sample records that were applied from this profile.
316 ///
317 /// This count does not include records from cold inlined callsites.
318 unsigned
319 SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS) const {
320  auto I = SampleCoverage.find(FS);
321 
322  // The size of the coverage map for FS represents the number of records
323  // that were marked used at least once.
324  unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;
325 
326  // If there are inlined callsites in this function, count the samples found
327  // in the respective bodies. However, do not bother counting callees with 0
328  // total samples, these are callees that were never invoked at runtime.
329  for (const auto &I : FS->getCallsiteSamples()) {
330  const FunctionSamples *CalleeSamples = &I.second;
331  if (callsiteIsHot(FS, CalleeSamples))
332  Count += countUsedRecords(CalleeSamples);
333  }
334 
335  return Count;
336 }
337 
338 /// Return the number of sample records in the body of this profile.
339 ///
340 /// This count does not include records from cold inlined callsites.
341 unsigned
342 SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS) const {
343  unsigned Count = FS->getBodySamples().size();
344 
345  // Only count records in hot callsites.
346  for (const auto &I : FS->getCallsiteSamples()) {
347  const FunctionSamples *CalleeSamples = &I.second;
348  if (callsiteIsHot(FS, CalleeSamples))
349  Count += countBodyRecords(CalleeSamples);
350  }
351 
352  return Count;
353 }
354 
355 /// Return the number of samples collected in the body of this profile.
356 ///
357 /// This count does not include samples from cold inlined callsites.
358 uint64_t
359 SampleCoverageTracker::countBodySamples(const FunctionSamples *FS) const {
360  uint64_t Total = 0;
361  for (const auto &I : FS->getBodySamples())
362  Total += I.second.getSamples();
363 
364  // Only count samples in hot callsites.
365  for (const auto &I : FS->getCallsiteSamples()) {
366  const FunctionSamples *CalleeSamples = &I.second;
367  if (callsiteIsHot(FS, CalleeSamples))
368  Total += countBodySamples(CalleeSamples);
369  }
370 
371  return Total;
372 }
373 
374 /// Return the fraction of sample records used in this profile.
375 ///
376 /// The returned value is an unsigned integer in the range 0-100 indicating
377 /// the percentage of sample records that were used while applying this
378 /// profile to the associated function.
379 unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
380  unsigned Total) const {
381  assert(Used <= Total &&
382  "number of used records cannot exceed the total number of records");
383  return Total > 0 ? Used * 100 / Total : 100;
384 }
385 
386 /// Clear all the per-function data used to load samples and propagate weights.
387 void SampleProfileLoader::clearFunctionData() {
388  BlockWeights.clear();
389  EdgeWeights.clear();
390  VisitedBlocks.clear();
391  VisitedEdges.clear();
392  EquivalenceClass.clear();
393  DT = nullptr;
394  PDT = nullptr;
395  LI = nullptr;
396  Predecessors.clear();
397  Successors.clear();
398  CoverageTracker.clear();
399 }
400 
401 /// \brief Returns the offset of lineno \p L to head_lineno \p H
402 ///
403 /// \param L Lineno
404 /// \param H Header lineno of the function
405 ///
406 /// \returns offset to the header lineno. 16 bits are used to represent offset.
407 /// We assume that a single function will not exceed 65535 LOC.
408 unsigned SampleProfileLoader::getOffset(unsigned L, unsigned H) const {
409  return (L - H) & 0xffff;
410 }
411 
412 /// \brief Print the weight of edge \p E on stream \p OS.
413 ///
414 /// \param OS Stream to emit the output to.
415 /// \param E Edge to print.
416 void SampleProfileLoader::printEdgeWeight(raw_ostream &OS, Edge E) {
417  OS << "weight[" << E.first->getName() << "->" << E.second->getName()
418  << "]: " << EdgeWeights[E] << "\n";
419 }
420 
421 /// \brief Print the equivalence class of block \p BB on stream \p OS.
422 ///
423 /// \param OS Stream to emit the output to.
424 /// \param BB Block to print.
425 void SampleProfileLoader::printBlockEquivalence(raw_ostream &OS,
426  const BasicBlock *BB) {
427  const BasicBlock *Equiv = EquivalenceClass[BB];
428  OS << "equivalence[" << BB->getName()
429  << "]: " << ((Equiv) ? EquivalenceClass[BB]->getName() : "NONE") << "\n";
430 }
431 
432 /// \brief Print the weight of block \p BB on stream \p OS.
433 ///
434 /// \param OS Stream to emit the output to.
435 /// \param BB Block to print.
436 void SampleProfileLoader::printBlockWeight(raw_ostream &OS,
437  const BasicBlock *BB) const {
438  const auto &I = BlockWeights.find(BB);
439  uint64_t W = (I == BlockWeights.end() ? 0 : I->second);
440  OS << "weight[" << BB->getName() << "]: " << W << "\n";
441 }
442 
443 /// \brief Get the weight for an instruction.
444 ///
445 /// The "weight" of an instruction \p Inst is the number of samples
446 /// collected on that instruction at runtime. To retrieve it, we
447 /// need to compute the line number of \p Inst relative to the start of its
448 /// function. We use HeaderLineno to compute the offset. We then
449 /// look up the samples collected for \p Inst using BodySamples.
450 ///
451 /// \param Inst Instruction to query.
452 ///
453 /// \returns the weight of \p Inst.
455 SampleProfileLoader::getInstWeight(const Instruction &Inst) {
456  const DebugLoc &DLoc = Inst.getDebugLoc();
457  if (!DLoc)
458  return std::error_code();
459 
460  const FunctionSamples *FS = findFunctionSamples(Inst);
461  if (!FS)
462  return std::error_code();
463 
464  // Ignore all intrinsics and branch instructions.
465  // Branch instruction usually contains debug info from sources outside of
466  // the residing basic block, thus we ignore them during annotation.
467  if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst))
468  return std::error_code();
469 
470  // If a call/invoke instruction is inlined in profile, but not inlined here,
471  // it means that the inlined callsite has no sample, thus the call
472  // instruction should have 0 count.
473  bool IsCall = isa<CallInst>(Inst) || isa<InvokeInst>(Inst);
474  if (IsCall && findCalleeFunctionSamples(Inst))
475  return 0;
476 
477  const DILocation *DIL = DLoc;
478  unsigned Lineno = DLoc.getLine();
479  unsigned HeaderLineno = DIL->getScope()->getSubprogram()->getLine();
480 
481  uint32_t LineOffset = getOffset(Lineno, HeaderLineno);
482  uint32_t Discriminator = DIL->getDiscriminator();
483  ErrorOr<uint64_t> R = IsCall
484  ? FS->findCallSamplesAt(LineOffset, Discriminator)
485  : FS->findSamplesAt(LineOffset, Discriminator);
486  if (R) {
487  bool FirstMark =
488  CoverageTracker.markSamplesUsed(FS, LineOffset, Discriminator, R.get());
489  if (FirstMark) {
490  const Function *F = Inst.getParent()->getParent();
491  LLVMContext &Ctx = F->getContext();
493  Ctx, DEBUG_TYPE, *F, DLoc,
494  Twine("Applied ") + Twine(*R) + " samples from profile (offset: " +
495  Twine(LineOffset) +
496  ((Discriminator) ? Twine(".") + Twine(Discriminator) : "") + ")");
497  }
498  DEBUG(dbgs() << " " << Lineno << "." << DIL->getDiscriminator() << ":"
499  << Inst << " (line offset: " << Lineno - HeaderLineno << "."
500  << DIL->getDiscriminator() << " - weight: " << R.get()
501  << ")\n");
502  }
503  return R;
504 }
505 
506 /// \brief Compute the weight of a basic block.
507 ///
508 /// The weight of basic block \p BB is the maximum weight of all the
509 /// instructions in BB.
510 ///
511 /// \param BB The basic block to query.
512 ///
513 /// \returns the weight for \p BB.
515 SampleProfileLoader::getBlockWeight(const BasicBlock *BB) {
516  uint64_t Max = 0;
517  bool HasWeight = false;
518  for (auto &I : BB->getInstList()) {
519  const ErrorOr<uint64_t> &R = getInstWeight(I);
520  if (R) {
521  Max = std::max(Max, R.get());
522  HasWeight = true;
523  }
524  }
525  return HasWeight ? ErrorOr<uint64_t>(Max) : std::error_code();
526 }
527 
528 /// \brief Compute and store the weights of every basic block.
529 ///
530 /// This populates the BlockWeights map by computing
531 /// the weights of every basic block in the CFG.
532 ///
533 /// \param F The function to query.
534 bool SampleProfileLoader::computeBlockWeights(Function &F) {
535  bool Changed = false;
536  DEBUG(dbgs() << "Block weights\n");
537  for (const auto &BB : F) {
538  ErrorOr<uint64_t> Weight = getBlockWeight(&BB);
539  if (Weight) {
540  BlockWeights[&BB] = Weight.get();
541  VisitedBlocks.insert(&BB);
542  Changed = true;
543  }
544  DEBUG(printBlockWeight(dbgs(), &BB));
545  }
546 
547  return Changed;
548 }
549 
550 /// \brief Get the FunctionSamples for a call instruction.
551 ///
552 /// The FunctionSamples of a call/invoke instruction \p Inst is the inlined
553 /// instance in which that call instruction is calling to. It contains
554 /// all samples that resides in the inlined instance. We first find the
555 /// inlined instance in which the call instruction is from, then we
556 /// traverse its children to find the callsite with the matching
557 /// location.
558 ///
559 /// \param Inst Call/Invoke instruction to query.
560 ///
561 /// \returns The FunctionSamples pointer to the inlined instance.
562 const FunctionSamples *
563 SampleProfileLoader::findCalleeFunctionSamples(const Instruction &Inst) const {
564  const DILocation *DIL = Inst.getDebugLoc();
565  if (!DIL) {
566  return nullptr;
567  }
568  DISubprogram *SP = DIL->getScope()->getSubprogram();
569  if (!SP)
570  return nullptr;
571 
572  const FunctionSamples *FS = findFunctionSamples(Inst);
573  if (FS == nullptr)
574  return nullptr;
575 
577  getOffset(DIL->getLine(), SP->getLine()), DIL->getDiscriminator()));
578 }
579 
580 /// \brief Get the FunctionSamples for an instruction.
581 ///
582 /// The FunctionSamples of an instruction \p Inst is the inlined instance
583 /// in which that instruction is coming from. We traverse the inline stack
584 /// of that instruction, and match it with the tree nodes in the profile.
585 ///
586 /// \param Inst Instruction to query.
587 ///
588 /// \returns the FunctionSamples pointer to the inlined instance.
589 const FunctionSamples *
590 SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
592  const DILocation *DIL = Inst.getDebugLoc();
593  if (!DIL) {
594  return Samples;
595  }
596  for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
597  DISubprogram *SP = DIL->getScope()->getSubprogram();
598  if (!SP)
599  return nullptr;
600  S.push_back(LineLocation(getOffset(DIL->getLine(), SP->getLine()),
601  DIL->getDiscriminator()));
602  }
603  if (S.size() == 0)
604  return Samples;
605  const FunctionSamples *FS = Samples;
606  for (int i = S.size() - 1; i >= 0 && FS != nullptr; i--) {
607  FS = FS->findFunctionSamplesAt(S[i]);
608  }
609  return FS;
610 }
611 
612 /// \brief Iteratively inline hot callsites of a function.
613 ///
614 /// Iteratively traverse all callsites of the function \p F, and find if
615 /// the corresponding inlined instance exists and is hot in profile. If
616 /// it is hot enough, inline the callsites and adds new callsites of the
617 /// callee into the caller.
618 ///
619 /// TODO: investigate the possibility of not invoking InlineFunction directly.
620 ///
621 /// \param F function to perform iterative inlining.
622 ///
623 /// \returns True if there is any inline happened.
624 bool SampleProfileLoader::inlineHotFunctions(Function &F) {
625  bool Changed = false;
626  LLVMContext &Ctx = F.getContext();
627  std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&](
628  Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); };
629  while (true) {
630  bool LocalChanged = false;
632  for (auto &BB : F) {
633  bool Hot = false;
635  for (auto &I : BB.getInstList()) {
636  const FunctionSamples *FS = nullptr;
637  if ((isa<CallInst>(I) || isa<InvokeInst>(I)) &&
638  (FS = findCalleeFunctionSamples(I))) {
639  Candidates.push_back(&I);
640  if (callsiteIsHot(Samples, FS))
641  Hot = true;
642  }
643  }
644  if (Hot) {
645  CIS.insert(CIS.begin(), Candidates.begin(), Candidates.end());
646  }
647  }
648  for (auto I : CIS) {
649  InlineFunctionInfo IFI(nullptr, ACT ? &GetAssumptionCache : nullptr);
650  CallSite CS(I);
651  Function *CalledFunction = CS.getCalledFunction();
652  if (!CalledFunction || !CalledFunction->getSubprogram())
653  continue;
654  DebugLoc DLoc = I->getDebugLoc();
655  uint64_t NumSamples = findCalleeFunctionSamples(*I)->getTotalSamples();
656  if (InlineFunction(CS, IFI)) {
657  LocalChanged = true;
658  emitOptimizationRemark(Ctx, DEBUG_TYPE, F, DLoc,
659  Twine("inlined hot callee '") +
660  CalledFunction->getName() + "' with " +
661  Twine(NumSamples) + " samples into '" +
662  F.getName() + "'");
663  }
664  }
665  if (LocalChanged) {
666  Changed = true;
667  } else {
668  break;
669  }
670  }
671  return Changed;
672 }
673 
674 /// \brief Find equivalence classes for the given block.
675 ///
676 /// This finds all the blocks that are guaranteed to execute the same
677 /// number of times as \p BB1. To do this, it traverses all the
678 /// descendants of \p BB1 in the dominator or post-dominator tree.
679 ///
680 /// A block BB2 will be in the same equivalence class as \p BB1 if
681 /// the following holds:
682 ///
683 /// 1- \p BB1 is a descendant of BB2 in the opposite tree. So, if BB2
684 /// is a descendant of \p BB1 in the dominator tree, then BB2 should
685 /// dominate BB1 in the post-dominator tree.
686 ///
687 /// 2- Both BB2 and \p BB1 must be in the same loop.
688 ///
689 /// For every block BB2 that meets those two requirements, we set BB2's
690 /// equivalence class to \p BB1.
691 ///
692 /// \param BB1 Block to check.
693 /// \param Descendants Descendants of \p BB1 in either the dom or pdom tree.
694 /// \param DomTree Opposite dominator tree. If \p Descendants is filled
695 /// with blocks from \p BB1's dominator tree, then
696 /// this is the post-dominator tree, and vice versa.
697 void SampleProfileLoader::findEquivalencesFor(
698  BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants,
700  const BasicBlock *EC = EquivalenceClass[BB1];
701  uint64_t Weight = BlockWeights[EC];
702  for (const auto *BB2 : Descendants) {
703  bool IsDomParent = DomTree->dominates(BB2, BB1);
704  bool IsInSameLoop = LI->getLoopFor(BB1) == LI->getLoopFor(BB2);
705  if (BB1 != BB2 && IsDomParent && IsInSameLoop) {
706  EquivalenceClass[BB2] = EC;
707  // If BB2 is visited, then the entire EC should be marked as visited.
708  if (VisitedBlocks.count(BB2)) {
709  VisitedBlocks.insert(EC);
710  }
711 
712  // If BB2 is heavier than BB1, make BB2 have the same weight
713  // as BB1.
714  //
715  // Note that we don't worry about the opposite situation here
716  // (when BB2 is lighter than BB1). We will deal with this
717  // during the propagation phase. Right now, we just want to
718  // make sure that BB1 has the largest weight of all the
719  // members of its equivalence set.
720  Weight = std::max(Weight, BlockWeights[BB2]);
721  }
722  }
723  if (EC == &EC->getParent()->getEntryBlock()) {
724  BlockWeights[EC] = Samples->getHeadSamples() + 1;
725  } else {
726  BlockWeights[EC] = Weight;
727  }
728 }
729 
730 /// \brief Find equivalence classes.
731 ///
732 /// Since samples may be missing from blocks, we can fill in the gaps by setting
733 /// the weights of all the blocks in the same equivalence class to the same
734 /// weight. To compute the concept of equivalence, we use dominance and loop
735 /// information. Two blocks B1 and B2 are in the same equivalence class if B1
736 /// dominates B2, B2 post-dominates B1 and both are in the same loop.
737 ///
738 /// \param F The function to query.
739 void SampleProfileLoader::findEquivalenceClasses(Function &F) {
740  SmallVector<BasicBlock *, 8> DominatedBBs;
741  DEBUG(dbgs() << "\nBlock equivalence classes\n");
742  // Find equivalence sets based on dominance and post-dominance information.
743  for (auto &BB : F) {
744  BasicBlock *BB1 = &BB;
745 
746  // Compute BB1's equivalence class once.
747  if (EquivalenceClass.count(BB1)) {
748  DEBUG(printBlockEquivalence(dbgs(), BB1));
749  continue;
750  }
751 
752  // By default, blocks are in their own equivalence class.
753  EquivalenceClass[BB1] = BB1;
754 
755  // Traverse all the blocks dominated by BB1. We are looking for
756  // every basic block BB2 such that:
757  //
758  // 1- BB1 dominates BB2.
759  // 2- BB2 post-dominates BB1.
760  // 3- BB1 and BB2 are in the same loop nest.
761  //
762  // If all those conditions hold, it means that BB2 is executed
763  // as many times as BB1, so they are placed in the same equivalence
764  // class by making BB2's equivalence class be BB1.
765  DominatedBBs.clear();
766  DT->getDescendants(BB1, DominatedBBs);
767  findEquivalencesFor(BB1, DominatedBBs, PDT.get());
768 
769  DEBUG(printBlockEquivalence(dbgs(), BB1));
770  }
771 
772  // Assign weights to equivalence classes.
773  //
774  // All the basic blocks in the same equivalence class will execute
775  // the same number of times. Since we know that the head block in
776  // each equivalence class has the largest weight, assign that weight
777  // to all the blocks in that equivalence class.
778  DEBUG(dbgs() << "\nAssign the same weight to all blocks in the same class\n");
779  for (auto &BI : F) {
780  const BasicBlock *BB = &BI;
781  const BasicBlock *EquivBB = EquivalenceClass[BB];
782  if (BB != EquivBB)
783  BlockWeights[BB] = BlockWeights[EquivBB];
784  DEBUG(printBlockWeight(dbgs(), BB));
785  }
786 }
787 
788 /// \brief Visit the given edge to decide if it has a valid weight.
789 ///
790 /// If \p E has not been visited before, we copy to \p UnknownEdge
791 /// and increment the count of unknown edges.
792 ///
793 /// \param E Edge to visit.
794 /// \param NumUnknownEdges Current number of unknown edges.
795 /// \param UnknownEdge Set if E has not been visited before.
796 ///
797 /// \returns E's weight, if known. Otherwise, return 0.
798 uint64_t SampleProfileLoader::visitEdge(Edge E, unsigned *NumUnknownEdges,
799  Edge *UnknownEdge) {
800  if (!VisitedEdges.count(E)) {
801  (*NumUnknownEdges)++;
802  *UnknownEdge = E;
803  return 0;
804  }
805 
806  return EdgeWeights[E];
807 }
808 
809 /// \brief Propagate weights through incoming/outgoing edges.
810 ///
811 /// If the weight of a basic block is known, and there is only one edge
812 /// with an unknown weight, we can calculate the weight of that edge.
813 ///
814 /// Similarly, if all the edges have a known count, we can calculate the
815 /// count of the basic block, if needed.
816 ///
817 /// \param F Function to process.
818 /// \param UpdateBlockCount Whether we should update basic block counts that
819 /// has already been annotated.
820 ///
821 /// \returns True if new weights were assigned to edges or blocks.
822 bool SampleProfileLoader::propagateThroughEdges(Function &F,
823  bool UpdateBlockCount) {
824  bool Changed = false;
825  DEBUG(dbgs() << "\nPropagation through edges\n");
826  for (const auto &BI : F) {
827  const BasicBlock *BB = &BI;
828  const BasicBlock *EC = EquivalenceClass[BB];
829 
830  // Visit all the predecessor and successor edges to determine
831  // which ones have a weight assigned already. Note that it doesn't
832  // matter that we only keep track of a single unknown edge. The
833  // only case we are interested in handling is when only a single
834  // edge is unknown (see setEdgeOrBlockWeight).
835  for (unsigned i = 0; i < 2; i++) {
836  uint64_t TotalWeight = 0;
837  unsigned NumUnknownEdges = 0, NumTotalEdges = 0;
838  Edge UnknownEdge, SelfReferentialEdge, SingleEdge;
839 
840  if (i == 0) {
841  // First, visit all predecessor edges.
842  NumTotalEdges = Predecessors[BB].size();
843  for (auto *Pred : Predecessors[BB]) {
844  Edge E = std::make_pair(Pred, BB);
845  TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
846  if (E.first == E.second)
847  SelfReferentialEdge = E;
848  }
849  if (NumTotalEdges == 1) {
850  SingleEdge = std::make_pair(Predecessors[BB][0], BB);
851  }
852  } else {
853  // On the second round, visit all successor edges.
854  NumTotalEdges = Successors[BB].size();
855  for (auto *Succ : Successors[BB]) {
856  Edge E = std::make_pair(BB, Succ);
857  TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
858  }
859  if (NumTotalEdges == 1) {
860  SingleEdge = std::make_pair(BB, Successors[BB][0]);
861  }
862  }
863 
864  // After visiting all the edges, there are three cases that we
865  // can handle immediately:
866  //
867  // - All the edge weights are known (i.e., NumUnknownEdges == 0).
868  // In this case, we simply check that the sum of all the edges
869  // is the same as BB's weight. If not, we change BB's weight
870  // to match. Additionally, if BB had not been visited before,
871  // we mark it visited.
872  //
873  // - Only one edge is unknown and BB has already been visited.
874  // In this case, we can compute the weight of the edge by
875  // subtracting the total block weight from all the known
876  // edge weights. If the edges weight more than BB, then the
877  // edge of the last remaining edge is set to zero.
878  //
879  // - There exists a self-referential edge and the weight of BB is
880  // known. In this case, this edge can be based on BB's weight.
881  // We add up all the other known edges and set the weight on
882  // the self-referential edge as we did in the previous case.
883  //
884  // In any other case, we must continue iterating. Eventually,
885  // all edges will get a weight, or iteration will stop when
886  // it reaches SampleProfileMaxPropagateIterations.
887  if (NumUnknownEdges <= 1) {
888  uint64_t &BBWeight = BlockWeights[EC];
889  if (NumUnknownEdges == 0) {
890  if (!VisitedBlocks.count(EC)) {
891  // If we already know the weight of all edges, the weight of the
892  // basic block can be computed. It should be no larger than the sum
893  // of all edge weights.
894  if (TotalWeight > BBWeight) {
895  BBWeight = TotalWeight;
896  Changed = true;
897  DEBUG(dbgs() << "All edge weights for " << BB->getName()
898  << " known. Set weight for block: ";
899  printBlockWeight(dbgs(), BB););
900  }
901  } else if (NumTotalEdges == 1 &&
902  EdgeWeights[SingleEdge] < BlockWeights[EC]) {
903  // If there is only one edge for the visited basic block, use the
904  // block weight to adjust edge weight if edge weight is smaller.
905  EdgeWeights[SingleEdge] = BlockWeights[EC];
906  Changed = true;
907  }
908  } else if (NumUnknownEdges == 1 && VisitedBlocks.count(EC)) {
909  // If there is a single unknown edge and the block has been
910  // visited, then we can compute E's weight.
911  if (BBWeight >= TotalWeight)
912  EdgeWeights[UnknownEdge] = BBWeight - TotalWeight;
913  else
914  EdgeWeights[UnknownEdge] = 0;
915  const BasicBlock *OtherEC;
916  if (i == 0)
917  OtherEC = EquivalenceClass[UnknownEdge.first];
918  else
919  OtherEC = EquivalenceClass[UnknownEdge.second];
920  // Edge weights should never exceed the BB weights it connects.
921  if (VisitedBlocks.count(OtherEC) &&
922  EdgeWeights[UnknownEdge] > BlockWeights[OtherEC])
923  EdgeWeights[UnknownEdge] = BlockWeights[OtherEC];
924  VisitedEdges.insert(UnknownEdge);
925  Changed = true;
926  DEBUG(dbgs() << "Set weight for edge: ";
927  printEdgeWeight(dbgs(), UnknownEdge));
928  }
929  } else if (VisitedBlocks.count(EC) && BlockWeights[EC] == 0) {
930  // If a block Weights 0, all its in/out edges should weight 0.
931  if (i == 0) {
932  for (auto *Pred : Predecessors[BB]) {
933  Edge E = std::make_pair(Pred, BB);
934  EdgeWeights[E] = 0;
935  VisitedEdges.insert(E);
936  }
937  } else {
938  for (auto *Succ : Successors[BB]) {
939  Edge E = std::make_pair(BB, Succ);
940  EdgeWeights[E] = 0;
941  VisitedEdges.insert(E);
942  }
943  }
944  } else if (SelfReferentialEdge.first && VisitedBlocks.count(EC)) {
945  uint64_t &BBWeight = BlockWeights[BB];
946  // We have a self-referential edge and the weight of BB is known.
947  if (BBWeight >= TotalWeight)
948  EdgeWeights[SelfReferentialEdge] = BBWeight - TotalWeight;
949  else
950  EdgeWeights[SelfReferentialEdge] = 0;
951  VisitedEdges.insert(SelfReferentialEdge);
952  Changed = true;
953  DEBUG(dbgs() << "Set self-referential edge weight to: ";
954  printEdgeWeight(dbgs(), SelfReferentialEdge));
955  }
956  if (UpdateBlockCount && !VisitedBlocks.count(EC) && TotalWeight > 0) {
957  BlockWeights[EC] = TotalWeight;
958  VisitedBlocks.insert(EC);
959  Changed = true;
960  }
961  }
962  }
963 
964  return Changed;
965 }
966 
967 /// \brief Build in/out edge lists for each basic block in the CFG.
968 ///
969 /// We are interested in unique edges. If a block B1 has multiple
970 /// edges to another block B2, we only add a single B1->B2 edge.
971 void SampleProfileLoader::buildEdges(Function &F) {
972  for (auto &BI : F) {
973  BasicBlock *B1 = &BI;
974 
975  // Add predecessors for B1.
977  if (!Predecessors[B1].empty())
978  llvm_unreachable("Found a stale predecessors list in a basic block.");
979  for (pred_iterator PI = pred_begin(B1), PE = pred_end(B1); PI != PE; ++PI) {
980  BasicBlock *B2 = *PI;
981  if (Visited.insert(B2).second)
982  Predecessors[B1].push_back(B2);
983  }
984 
985  // Add successors for B1.
986  Visited.clear();
987  if (!Successors[B1].empty())
988  llvm_unreachable("Found a stale successors list in a basic block.");
989  for (succ_iterator SI = succ_begin(B1), SE = succ_end(B1); SI != SE; ++SI) {
990  BasicBlock *B2 = *SI;
991  if (Visited.insert(B2).second)
992  Successors[B1].push_back(B2);
993  }
994  }
995 }
996 
997 /// \brief Propagate weights into edges
998 ///
999 /// The following rules are applied to every block BB in the CFG:
1000 ///
1001 /// - If BB has a single predecessor/successor, then the weight
1002 /// of that edge is the weight of the block.
1003 ///
1004 /// - If all incoming or outgoing edges are known except one, and the
1005 /// weight of the block is already known, the weight of the unknown
1006 /// edge will be the weight of the block minus the sum of all the known
1007 /// edges. If the sum of all the known edges is larger than BB's weight,
1008 /// we set the unknown edge weight to zero.
1009 ///
1010 /// - If there is a self-referential edge, and the weight of the block is
1011 /// known, the weight for that edge is set to the weight of the block
1012 /// minus the weight of the other incoming edges to that block (if
1013 /// known).
1014 void SampleProfileLoader::propagateWeights(Function &F) {
1015  bool Changed = true;
1016  unsigned I = 0;
1017 
1018  // Add an entry count to the function using the samples gathered
1019  // at the function entry.
1020  F.setEntryCount(Samples->getHeadSamples() + 1);
1021 
1022  // If BB weight is larger than its corresponding loop's header BB weight,
1023  // use the BB weight to replace the loop header BB weight.
1024  for (auto &BI : F) {
1025  BasicBlock *BB = &BI;
1026  Loop *L = LI->getLoopFor(BB);
1027  if (!L) {
1028  continue;
1029  }
1030  BasicBlock *Header = L->getHeader();
1031  if (Header && BlockWeights[BB] > BlockWeights[Header]) {
1032  BlockWeights[Header] = BlockWeights[BB];
1033  }
1034  }
1035 
1036  // Before propagation starts, build, for each block, a list of
1037  // unique predecessors and successors. This is necessary to handle
1038  // identical edges in multiway branches. Since we visit all blocks and all
1039  // edges of the CFG, it is cleaner to build these lists once at the start
1040  // of the pass.
1041  buildEdges(F);
1042 
1043  // Propagate until we converge or we go past the iteration limit.
1044  while (Changed && I++ < SampleProfileMaxPropagateIterations) {
1045  Changed = propagateThroughEdges(F, false);
1046  }
1047 
1048  // The first propagation propagates BB counts from annotated BBs to unknown
1049  // BBs. The 2nd propagation pass resets edges weights, and use all BB weights
1050  // to propagate edge weights.
1051  VisitedEdges.clear();
1052  Changed = true;
1053  while (Changed && I++ < SampleProfileMaxPropagateIterations) {
1054  Changed = propagateThroughEdges(F, false);
1055  }
1056 
1057  // The 3rd propagation pass allows adjust annotated BB weights that are
1058  // obviously wrong.
1059  Changed = true;
1060  while (Changed && I++ < SampleProfileMaxPropagateIterations) {
1061  Changed = propagateThroughEdges(F, true);
1062  }
1063 
1064  // Generate MD_prof metadata for every branch instruction using the
1065  // edge weights computed during propagation.
1066  DEBUG(dbgs() << "\nPropagation complete. Setting branch weights\n");
1067  LLVMContext &Ctx = F.getContext();
1068  MDBuilder MDB(Ctx);
1069  for (auto &BI : F) {
1070  BasicBlock *BB = &BI;
1071 
1072  if (BlockWeights[BB]) {
1073  for (auto &I : BB->getInstList()) {
1074  if (CallInst *CI = dyn_cast<CallInst>(&I)) {
1075  if (!dyn_cast<IntrinsicInst>(&I)) {
1076  SmallVector<uint32_t, 1> Weights;
1077  Weights.push_back(BlockWeights[BB]);
1078  CI->setMetadata(LLVMContext::MD_prof,
1079  MDB.createBranchWeights(Weights));
1080  }
1081  }
1082  }
1083  }
1084  TerminatorInst *TI = BB->getTerminator();
1085  if (TI->getNumSuccessors() == 1)
1086  continue;
1087  if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
1088  continue;
1089 
1090  DEBUG(dbgs() << "\nGetting weights for branch at line "
1091  << TI->getDebugLoc().getLine() << ".\n");
1092  SmallVector<uint32_t, 4> Weights;
1093  uint32_t MaxWeight = 0;
1094  DebugLoc MaxDestLoc;
1095  for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) {
1096  BasicBlock *Succ = TI->getSuccessor(I);
1097  Edge E = std::make_pair(BB, Succ);
1098  uint64_t Weight = EdgeWeights[E];
1099  DEBUG(dbgs() << "\t"; printEdgeWeight(dbgs(), E));
1100  // Use uint32_t saturated arithmetic to adjust the incoming weights,
1101  // if needed. Sample counts in profiles are 64-bit unsigned values,
1102  // but internally branch weights are expressed as 32-bit values.
1103  if (Weight > std::numeric_limits<uint32_t>::max()) {
1104  DEBUG(dbgs() << " (saturated due to uint32_t overflow)");
1105  Weight = std::numeric_limits<uint32_t>::max();
1106  }
1107  // Weight is added by one to avoid propagation errors introduced by
1108  // 0 weights.
1109  Weights.push_back(static_cast<uint32_t>(Weight + 1));
1110  if (Weight != 0) {
1111  if (Weight > MaxWeight) {
1112  MaxWeight = Weight;
1113  MaxDestLoc = Succ->getFirstNonPHIOrDbgOrLifetime()->getDebugLoc();
1114  }
1115  }
1116  }
1117 
1118  // Only set weights if there is at least one non-zero weight.
1119  // In any other case, let the analyzer set weights.
1120  if (MaxWeight > 0) {
1121  DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n");
1123  MDB.createBranchWeights(Weights));
1124  DebugLoc BranchLoc = TI->getDebugLoc();
1126  Ctx, DEBUG_TYPE, F, MaxDestLoc,
1127  Twine("most popular destination for conditional branches at ") +
1128  ((BranchLoc) ? Twine(BranchLoc->getFilename() + ":" +
1129  Twine(BranchLoc.getLine()) + ":" +
1130  Twine(BranchLoc.getCol()))
1131  : Twine("<UNKNOWN LOCATION>")));
1132  } else {
1133  DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n");
1134  }
1135  }
1136 }
1137 
1138 /// \brief Get the line number for the function header.
1139 ///
1140 /// This looks up function \p F in the current compilation unit and
1141 /// retrieves the line number where the function is defined. This is
1142 /// line 0 for all the samples read from the profile file. Every line
1143 /// number is relative to this line.
1144 ///
1145 /// \param F Function object to query.
1146 ///
1147 /// \returns the line number where \p F is defined. If it returns 0,
1148 /// it means that there is no debug information available for \p F.
1149 unsigned SampleProfileLoader::getFunctionLoc(Function &F) {
1150  if (DISubprogram *S = F.getSubprogram())
1151  return S->getLine();
1152 
1153  // If the start of \p F is missing, emit a diagnostic to inform the user
1154  // about the missed opportunity.
1156  "No debug information found in function " + F.getName() +
1157  ": Function profile not used",
1158  DS_Warning));
1159  return 0;
1160 }
1161 
1162 void SampleProfileLoader::computeDominanceAndLoopInfo(Function &F) {
1163  DT.reset(new DominatorTree);
1164  DT->recalculate(F);
1165 
1166  PDT.reset(new DominatorTreeBase<BasicBlock>(true));
1167  PDT->recalculate(F);
1168 
1169  LI.reset(new LoopInfo);
1170  LI->analyze(*DT);
1171 }
1172 
1173 /// \brief Generate branch weight metadata for all branches in \p F.
1174 ///
1175 /// Branch weights are computed out of instruction samples using a
1176 /// propagation heuristic. Propagation proceeds in 3 phases:
1177 ///
1178 /// 1- Assignment of block weights. All the basic blocks in the function
1179 /// are initial assigned the same weight as their most frequently
1180 /// executed instruction.
1181 ///
1182 /// 2- Creation of equivalence classes. Since samples may be missing from
1183 /// blocks, we can fill in the gaps by setting the weights of all the
1184 /// blocks in the same equivalence class to the same weight. To compute
1185 /// the concept of equivalence, we use dominance and loop information.
1186 /// Two blocks B1 and B2 are in the same equivalence class if B1
1187 /// dominates B2, B2 post-dominates B1 and both are in the same loop.
1188 ///
1189 /// 3- Propagation of block weights into edges. This uses a simple
1190 /// propagation heuristic. The following rules are applied to every
1191 /// block BB in the CFG:
1192 ///
1193 /// - If BB has a single predecessor/successor, then the weight
1194 /// of that edge is the weight of the block.
1195 ///
1196 /// - If all the edges are known except one, and the weight of the
1197 /// block is already known, the weight of the unknown edge will
1198 /// be the weight of the block minus the sum of all the known
1199 /// edges. If the sum of all the known edges is larger than BB's weight,
1200 /// we set the unknown edge weight to zero.
1201 ///
1202 /// - If there is a self-referential edge, and the weight of the block is
1203 /// known, the weight for that edge is set to the weight of the block
1204 /// minus the weight of the other incoming edges to that block (if
1205 /// known).
1206 ///
1207 /// Since this propagation is not guaranteed to finalize for every CFG, we
1208 /// only allow it to proceed for a limited number of iterations (controlled
1209 /// by -sample-profile-max-propagate-iterations).
1210 ///
1211 /// FIXME: Try to replace this propagation heuristic with a scheme
1212 /// that is guaranteed to finalize. A work-list approach similar to
1213 /// the standard value propagation algorithm used by SSA-CCP might
1214 /// work here.
1215 ///
1216 /// Once all the branch weights are computed, we emit the MD_prof
1217 /// metadata on BB using the computed values for each of its branches.
1218 ///
1219 /// \param F The function to query.
1220 ///
1221 /// \returns true if \p F was modified. Returns false, otherwise.
1222 bool SampleProfileLoader::emitAnnotations(Function &F) {
1223  bool Changed = false;
1224 
1225  if (getFunctionLoc(F) == 0)
1226  return false;
1227 
1228  DEBUG(dbgs() << "Line number for the first instruction in " << F.getName()
1229  << ": " << getFunctionLoc(F) << "\n");
1230 
1231  Changed |= inlineHotFunctions(F);
1232 
1233  // Compute basic block weights.
1234  Changed |= computeBlockWeights(F);
1235 
1236  if (Changed) {
1237  // Compute dominance and loop info needed for propagation.
1238  computeDominanceAndLoopInfo(F);
1239 
1240  // Find equivalence classes.
1241  findEquivalenceClasses(F);
1242 
1243  // Propagate weights to all edges.
1244  propagateWeights(F);
1245  }
1246 
1247  // If coverage checking was requested, compute it now.
1249  unsigned Used = CoverageTracker.countUsedRecords(Samples);
1250  unsigned Total = CoverageTracker.countBodyRecords(Samples);
1251  unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
1252  if (Coverage < SampleProfileRecordCoverage) {
1254  F.getSubprogram()->getFilename(), getFunctionLoc(F),
1255  Twine(Used) + " of " + Twine(Total) + " available profile records (" +
1256  Twine(Coverage) + "%) were applied",
1257  DS_Warning));
1258  }
1259  }
1260 
1262  uint64_t Used = CoverageTracker.getTotalUsedSamples();
1263  uint64_t Total = CoverageTracker.countBodySamples(Samples);
1264  unsigned Coverage = CoverageTracker.computeCoverage(Used, Total);
1265  if (Coverage < SampleProfileSampleCoverage) {
1267  F.getSubprogram()->getFilename(), getFunctionLoc(F),
1268  Twine(Used) + " of " + Twine(Total) + " available profile samples (" +
1269  Twine(Coverage) + "%) were applied",
1270  DS_Warning));
1271  }
1272  }
1273  return Changed;
1274 }
1275 
1277 INITIALIZE_PASS_BEGIN(SampleProfileLoaderLegacyPass, "sample-profile",
1278  "Sample Profile loader", false, false)
1280 INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass, "sample-profile",
1281  "Sample Profile loader", false, false)
1282 
1283 bool SampleProfileLoader::doInitialization(Module &M) {
1284  auto &Ctx = M.getContext();
1285  auto ReaderOrErr = SampleProfileReader::create(Filename, Ctx);
1286  if (std::error_code EC = ReaderOrErr.getError()) {
1287  std::string Msg = "Could not open profile: " + EC.message();
1288  Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1289  return false;
1290  }
1291  Reader = std::move(ReaderOrErr.get());
1292  ProfileIsValid = (Reader->read() == sampleprof_error::success);
1293  return true;
1294 }
1295 
1297  return new SampleProfileLoaderLegacyPass(SampleProfileFile);
1298 }
1299 
1301  return new SampleProfileLoaderLegacyPass(Name);
1302 }
1303 
1304 bool SampleProfileLoader::runOnModule(Module &M) {
1305  if (!ProfileIsValid)
1306  return false;
1307 
1308  // Compute the total number of samples collected in this profile.
1309  for (const auto &I : Reader->getProfiles())
1310  TotalCollectedSamples += I.second.getTotalSamples();
1311 
1312  bool retval = false;
1313  for (auto &F : M)
1314  if (!F.isDeclaration()) {
1315  clearFunctionData();
1316  retval |= runOnFunction(F);
1317  }
1318  if (M.getProfileSummary() == nullptr)
1319  M.setProfileSummary(Reader->getSummary().getMD(M.getContext()));
1320  return retval;
1321 }
1322 
1323 bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) {
1324  // FIXME: pass in AssumptionCache correctly for the new pass manager.
1325  SampleLoader.setACT(&getAnalysis<AssumptionCacheTracker>());
1326  return SampleLoader.runOnModule(M);
1327 }
1328 
1329 bool SampleProfileLoader::runOnFunction(Function &F) {
1330  F.setEntryCount(0);
1331  Samples = Reader->getSamplesFor(F);
1332  if (!Samples->empty())
1333  return emitAnnotations(F);
1334  return false;
1335 }
1336 
1338  ModuleAnalysisManager &AM) {
1339 
1340  SampleProfileLoader SampleLoader(SampleProfileFile);
1341 
1342  SampleLoader.doInitialization(M);
1343 
1344  if (!SampleLoader.runOnModule(M))
1345  return PreservedAnalyses::all();
1346 
1347  return PreservedAnalyses::none();
1348 }
MachineLoop * L
Represents either an error or a value T.
Definition: ErrorOr.h:68
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:226
const BodySampleMap & getBodySamples() const
Return all the samples collected in the body of the function.
Definition: SampleProf.h:265
size_t i
sample profile
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
This class represents a function call, abstracting a target machine's calling convention.
This file contains the declarations for metadata subclasses.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of .assume calls within a function.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(const Twine &Filename, LLVMContext &C)
Create a sample profile reader appropriate to the file format.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:100
const FunctionSamples * findFunctionSamplesAt(const LineLocation &Loc) const
Return a pointer to function samples at the given callsite location.
Definition: SampleProf.h:246
A debug info location.
Definition: DebugLoc.h:34
InlineFunctionInfo - This class captures the data input to the InlineFunction call, and records the auxiliary results produced by it.
Definition: Cloning.h:177
void emitOptimizationRemark(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg)
Emit an optimization-applied message.
BlockT * getHeader() const
Definition: LoopInfo.h:102
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
Representation of the samples collected for a function.
Definition: SampleProf.h:181
bool InlineFunction(CallInst *C, InlineFunctionInfo &IFI, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true)
InlineFunction - This function inlines the called function into the basic block of the caller...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static StringRef getName(Value *V)
StringRef getFilename() const
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:106
Subprogram description.
ErrorOr< uint64_t > findCallSamplesAt(uint32_t LineOffset, uint32_t Discriminator) const
Return the total number of call target samples collected at a given location.
Definition: SampleProf.h:228
#define F(x, y, z)
Definition: MD5.cpp:51
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
static cl::opt< std::string > SampleProfileFile("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden)
Debug location.
ModulePass * createSampleProfileLoaderPass()
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:96
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:110
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:109
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
Definition: InstrTypes.h:74
unsigned getLine() const
Definition: DebugLoc.cpp:25
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:52
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:107
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
Definition: InstrTypes.h:79
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
This file contains the declarations for the subclasses of Constant, which represent the different fla...
#define H(x, y, z)
Definition: MD5.cpp:53
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:116
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:259
Represent the analysis usage information of a pass.
const InstListType & getInstList() const
Return the underlying instruction list container.
Definition: BasicBlock.h:249
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:119
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
Definition: SampleProf.h:258
void initializeSampleProfileLoaderLegacyPassPass(PassRegistry &)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:113
Instruction * getFirstNonPHIOrDbgOrLifetime()
Returns a pointer to the first instruction in this block that is not a PHINode, a debug intrinsic...
Definition: BasicBlock.cpp:194
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1183
unsigned getCol() const
Definition: DebugLoc.cpp:30
void setEntryCount(uint64_t Count)
Set the entry count for this function.
Definition: Function.cpp:1282
const CallsiteSampleMap & getCallsiteSamples() const
Return all the callsite samples collected in the body of the function.
Definition: SampleProf.h:268
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
static cl::opt< unsigned > SampleProfileRecordCoverage("sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"), cl::desc("Emit a warning if less than N% of records in the input profile ""are matched to the IR."))
Module.h This file contains the declarations for the Module class.
const BasicBlock & getEntryBlock() const
Definition: Function.h:519
static cl::opt< unsigned > SampleProfileMaxPropagateIterations("sample-profile-max-propagate-iterations", cl::init(100), cl::desc("Maximum number of iterations to go through when propagating ""sample block/edge weights through the CFG."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1458
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:191
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:464
static cl::opt< double > SampleProfileHotThreshold("sample-profile-inline-hot-threshold", cl::init(0.1), cl::value_desc("N"), cl::desc("Inlined functions that account for more than N% of all samples ""collected in the parent function, will be inlined again."))
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
#define DEBUG_TYPE
Represents the relative location of an instruction.
Definition: SampleProf.h:88
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:188
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:368
static cl::opt< unsigned > SampleProfileSampleCoverage("sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"), cl::desc("Emit a warning if less than N% of samples in the input profile ""are matched to the IR."))
#define I(x, y, z)
Definition: MD5.cpp:54
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:235
sample Sample Profile false
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
INITIALIZE_PASS_BEGIN(SampleProfileLoaderLegacyPass,"sample-profile","Sample Profile loader", false, false) INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass
Provides ErrorOr<T> smart pointer.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
sample Sample Profile loader
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:44
#define DEBUG(X)
Definition: Debug.h:100
ErrorOr< uint64_t > findSamplesAt(uint32_t LineOffset, uint32_t Discriminator) const
Return the number of samples collected at the given location.
Definition: SampleProf.h:216
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
A container for analyses that lazily runs them and caches their results.
T Max(T a, T b)
Definition: FuzzerDefs.h:57
Diagnostic information for the sample profiler.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
const BasicBlock * getParent() const
Definition: Instruction.h:62
This file provides the interface for the sampled PGO loader pass.
reference get()
Definition: ErrorOr.h:166