LLVM  16.0.0git
InlineAdvisor.cpp
Go to the documentation of this file.
1 //===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10 // related types.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/Statistic.h"
25 #include "llvm/IR/PassManager.h"
28 
29 using namespace llvm;
30 #define DEBUG_TYPE "inline"
31 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
32 #define LLVM_HAVE_TF_AOT
33 #endif
34 
35 // This weirdly named statistic tracks the number of times that, when attempting
36 // to inline a function A into B, we analyze the callers of B in order to see
37 // if those would be more profitable and blocked inline steps.
38 STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
39 
40 /// Flag to add inline messages as callsite attributes 'inline-remark'.
41 static cl::opt<bool>
42  InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
43  cl::Hidden,
44  cl::desc("Enable adding inline-remark attribute to"
45  " callsites processed by inliner but decided"
46  " to be not inlined"));
47 
48 static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
49  cl::Hidden,
50  cl::desc("Enable deferred inlining"));
51 
52 // An integer used to limit the cost of inline deferral. The default negative
53 // number tells shouldBeDeferred to only take the secondary cost into account.
54 static cl::opt<int>
55  InlineDeferralScale("inline-deferral-scale",
56  cl::desc("Scale to limit the cost of inline deferral"),
57  cl::init(2), cl::Hidden);
58 
59 static cl::opt<bool>
60  AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
61  cl::desc("If true, annotate inline advisor remarks "
62  "with LTO and pass information."));
63 
64 namespace llvm {
66 }
67 
68 namespace {
69 using namespace llvm::ore;
70 class MandatoryInlineAdvice : public InlineAdvice {
71 public:
72  MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
74  bool IsInliningMandatory)
75  : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
76 
77 private:
78  void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
79 
80  void recordInliningImpl() override {
81  if (IsInliningRecommended)
82  emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
84  Remark << ": always inline attribute";
85  });
86  }
87 
88  void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
89  if (IsInliningRecommended)
90  ORE.emit([&]() {
92  "NotInlined", DLoc, Block)
93  << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
94  << NV("Caller", Caller)
95  << "': " << NV("Reason", Result.getFailureReason());
96  });
97  }
98 
99  void recordUnattemptedInliningImpl() override {
100  assert(!IsInliningRecommended && "Expected to attempt inlining");
101  }
102 };
103 } // namespace
104 
105 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
106  const InlineResult &Result) {
107  using namespace ore;
108  llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
109  "; " + inlineCostStr(*OIC));
110  ORE.emit([&]() {
112  "NotInlined", DLoc, Block)
113  << "'" << NV("Callee", Callee) << "' is not inlined into '"
114  << NV("Caller", Caller)
115  << "': " << NV("Reason", Result.getFailureReason());
116  });
117 }
118 
119 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
120  if (EmitRemarks)
122  /* ForProfileContext= */ false,
124 }
125 
126 void DefaultInlineAdvice::recordInliningImpl() {
127  if (EmitRemarks)
129  /* ForProfileContext= */ false,
131 }
132 
134  CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
135  Function &Caller = *CB.getCaller();
136  ProfileSummaryInfo *PSI =
138  .getCachedResult<ProfileSummaryAnalysis>(
139  *CB.getParent()->getParent()->getParent());
140 
142  auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
144  };
145  auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
147  };
148  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
150  };
151 
152  auto GetInlineCost = [&](CallBase &CB) {
154  auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
155  bool RemarksEnabled =
157  DEBUG_TYPE);
158  return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
159  GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
160  };
161  return llvm::shouldInline(
162  CB, GetInlineCost, ORE,
163  Params.EnableDeferral.value_or(EnableInlineDeferral));
164 }
165 
166 std::unique_ptr<InlineAdvice>
167 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
168  auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
169  return std::make_unique<DefaultInlineAdvice>(
170  this, CB, OIC,
172 }
173 
177  : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
178  DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
180 
181 void InlineAdvice::recordInlineStatsIfNeeded() {
183  Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
184 }
185 
187  markRecorded();
188  recordInlineStatsIfNeeded();
190 }
191 
193  markRecorded();
194  recordInlineStatsIfNeeded();
196 }
197 
199 
202  const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
203  auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
204  switch (Mode) {
206  LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
207  Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
208  // Restrict replay to default advisor, ML advisors are stateful so
209  // replay will need augmentations to interleave with them correctly.
210  if (!ReplaySettings.ReplayFile.empty()) {
211  Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(),
212  std::move(Advisor), ReplaySettings,
213  /* EmitRemarks =*/true, IC);
214  }
215  break;
217 #ifdef LLVM_HAVE_TF_API
218  LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
219  Advisor =
220  llvm::getDevelopmentModeAdvisor(M, MAM, [&FAM, Params](CallBase &CB) {
221  auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
222  return OIC.has_value();
223  });
224 #endif
225  break;
227 #ifdef LLVM_HAVE_TF_AOT
228  LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
229  Advisor = llvm::getReleaseModeAdvisor(M, MAM);
230 #endif
231  break;
232  }
233 
234  return !!Advisor;
235 }
236 
237 /// Return true if inlining of CB can block the caller from being
238 /// inlined which is proved to be more beneficial. \p IC is the
239 /// estimated inline cost associated with callsite \p CB.
240 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the
241 /// caller if \p CB is suppressed for inlining.
242 static bool
243 shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
244  function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
245  // For now we only handle local or inline functions.
246  if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
247  return false;
248  // If the cost of inlining CB is non-positive, it is not going to prevent the
249  // caller from being inlined into its callers and hence we don't need to
250  // defer.
251  if (IC.getCost() <= 0)
252  return false;
253  // Try to detect the case where the current inlining candidate caller (call
254  // it B) is a static or linkonce-ODR function and is an inlining candidate
255  // elsewhere, and the current candidate callee (call it C) is large enough
256  // that inlining it into B would make B too big to inline later. In these
257  // circumstances it may be best not to inline C into B, but to inline B into
258  // its callers.
259  //
260  // This only applies to static and linkonce-ODR functions because those are
261  // expected to be available for inlining in the translation units where they
262  // are used. Thus we will always have the opportunity to make local inlining
263  // decisions. Importantly the linkonce-ODR linkage covers inline functions
264  // and templates in C++.
265  //
266  // FIXME: All of this logic should be sunk into getInlineCost. It relies on
267  // the internal implementation of the inline cost metrics rather than
268  // treating them as truly abstract units etc.
269  TotalSecondaryCost = 0;
270  // The candidate cost to be imposed upon the current function.
271  int CandidateCost = IC.getCost() - 1;
272  // If the caller has local linkage and can be inlined to all its callers, we
273  // can apply a huge negative bonus to TotalSecondaryCost.
274  bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
275  // This bool tracks what happens if we DO inline C into B.
276  bool InliningPreventsSomeOuterInline = false;
277  unsigned NumCallerUsers = 0;
278  for (User *U : Caller->users()) {
279  CallBase *CS2 = dyn_cast<CallBase>(U);
280 
281  // If this isn't a call to Caller (it could be some other sort
282  // of reference) skip it. Such references will prevent the caller
283  // from being removed.
284  if (!CS2 || CS2->getCalledFunction() != Caller) {
285  ApplyLastCallBonus = false;
286  continue;
287  }
288 
289  InlineCost IC2 = GetInlineCost(*CS2);
290  ++NumCallerCallersAnalyzed;
291  if (!IC2) {
292  ApplyLastCallBonus = false;
293  continue;
294  }
295  if (IC2.isAlways())
296  continue;
297 
298  // See if inlining of the original callsite would erase the cost delta of
299  // this callsite. We subtract off the penalty for the call instruction,
300  // which we would be deleting.
301  if (IC2.getCostDelta() <= CandidateCost) {
302  InliningPreventsSomeOuterInline = true;
303  TotalSecondaryCost += IC2.getCost();
304  NumCallerUsers++;
305  }
306  }
307 
308  if (!InliningPreventsSomeOuterInline)
309  return false;
310 
311  // If all outer calls to Caller would get inlined, the cost for the last
312  // one is set very low by getInlineCost, in anticipation that Caller will
313  // be removed entirely. We did not account for this above unless there
314  // is only one caller of Caller.
315  if (ApplyLastCallBonus)
316  TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
317 
318  // If InlineDeferralScale is negative, then ignore the cost of primary
319  // inlining -- IC.getCost() multiplied by the number of callers to Caller.
320  if (InlineDeferralScale < 0)
321  return TotalSecondaryCost < IC.getCost();
322 
323  int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
324  int Allowance = IC.getCost() * InlineDeferralScale;
325  return TotalCost < Allowance;
326 }
327 
328 namespace llvm {
330  return R << Arg.Val;
331 }
332 
333 template <class RemarkT>
334 RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
335  using namespace ore;
336  if (IC.isAlways()) {
337  R << "(cost=always)";
338  } else if (IC.isNever()) {
339  R << "(cost=never)";
340  } else {
341  R << "(cost=" << ore::NV("Cost", IC.getCost())
342  << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
343  }
344  if (const char *Reason = IC.getReason())
345  R << ": " << ore::NV("Reason", Reason);
346  return R;
347 }
348 } // namespace llvm
349 
350 std::string llvm::inlineCostStr(const InlineCost &IC) {
351  std::string Buffer;
352  raw_string_ostream Remark(Buffer);
353  Remark << IC;
354  return Remark.str();
355 }
356 
359  return;
360 
361  Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
362  CB.addFnAttr(Attr);
363 }
364 
365 /// Return the cost only if the inliner should attempt to inline at the given
366 /// CallSite. If we return the cost, we will emit an optimisation remark later
367 /// using that cost, so we won't do so from this function. Return std::nullopt
368 /// if inlining should not be attempted.
371  function_ref<InlineCost(CallBase &CB)> GetInlineCost,
372  OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
373  using namespace ore;
374 
375  InlineCost IC = GetInlineCost(CB);
376  Instruction *Call = &CB;
378  Function *Caller = CB.getCaller();
379 
380  if (IC.isAlways()) {
381  LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
382  << ", Call: " << CB << "\n");
383  return IC;
384  }
385 
386  if (!IC) {
387  LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
388  << ", Call: " << CB << "\n");
389  if (IC.isNever()) {
390  ORE.emit([&]() {
391  return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
392  << "'" << NV("Callee", Callee) << "' not inlined into '"
393  << NV("Caller", Caller)
394  << "' because it should never be inlined " << IC;
395  });
396  } else {
397  ORE.emit([&]() {
398  return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
399  << "'" << NV("Callee", Callee) << "' not inlined into '"
400  << NV("Caller", Caller) << "' because too costly to inline "
401  << IC;
402  });
403  }
405  return std::nullopt;
406  }
407 
408  int TotalSecondaryCost = 0;
409  if (EnableDeferral &&
410  shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
411  LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
412  << " Cost = " << IC.getCost()
413  << ", outer Cost = " << TotalSecondaryCost << '\n');
414  ORE.emit([&]() {
415  return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
416  Call)
417  << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
418  << "' increases the cost of inlining '" << NV("Caller", Caller)
419  << "' in other contexts";
420  });
421  setInlineRemark(CB, "deferred");
422  return std::nullopt;
423  }
424 
425  LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
426  << '\n');
427  return IC;
428 }
429 
431  const CallSiteFormat &Format) {
432  std::string Buffer;
433  raw_string_ostream CallSiteLoc(Buffer);
434  bool First = true;
435  for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
436  if (!First)
437  CallSiteLoc << " @ ";
438  // Note that negative line offset is actually possible, but we use
439  // unsigned int to match line offset representation in remarks so
440  // it's directly consumable by relay advisor.
441  uint32_t Offset =
442  DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
443  uint32_t Discriminator = DIL->getBaseDiscriminator();
444  StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
445  if (Name.empty())
446  Name = DIL->getScope()->getSubprogram()->getName();
447  CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
448  if (Format.outputColumn())
449  CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
450  if (Format.outputDiscriminator() && Discriminator)
451  CallSiteLoc << "." << llvm::utostr(Discriminator);
452  First = false;
453  }
454 
455  return CallSiteLoc.str();
456 }
457 
459  if (!DLoc) {
460  return;
461  }
462 
463  bool First = true;
464  Remark << " at callsite ";
465  for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
466  if (!First)
467  Remark << " @ ";
468  unsigned int Offset = DIL->getLine();
469  Offset -= DIL->getScope()->getSubprogram()->getLine();
470  unsigned int Discriminator = DIL->getBaseDiscriminator();
471  StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
472  if (Name.empty())
473  Name = DIL->getScope()->getSubprogram()->getName();
474  Remark << Name << ":" << ore::NV("Line", Offset) << ":"
475  << ore::NV("Column", DIL->getColumn());
476  if (Discriminator)
477  Remark << "." << ore::NV("Disc", Discriminator);
478  First = false;
479  }
480 
481  Remark << ";";
482 }
483 
485  OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
486  const Function &Callee, const Function &Caller, bool AlwaysInline,
487  function_ref<void(OptimizationRemark &)> ExtraContext,
488  const char *PassName) {
489  ORE.emit([&]() {
490  StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
492  DLoc, Block);
493  Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
494  << ore::NV("Caller", &Caller) << "'";
495  if (ExtraContext)
496  ExtraContext(Remark);
498  return Remark;
499  });
500 }
501 
503  OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
504  const Function &Callee, const Function &Caller, const InlineCost &IC,
505  bool ForProfileContext, const char *PassName) {
507  ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
508  [&](OptimizationRemark &Remark) {
509  if (ForProfileContext)
510  Remark << " to match profiling context";
511  Remark << " with " << IC;
512  },
513  PassName);
514 }
515 
518  : M(M), FAM(FAM), IC(IC),
519  AnnotatedInlinePassName((IC && AnnotateInlinePhase)
521  : DEBUG_TYPE) {
524  std::make_unique<ImportedFunctionsInliningStatistics>();
525  ImportedFunctionsStats->setModuleInfo(M);
526  }
527 }
528 
534  }
535 }
536 
537 std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
538  bool Advice) {
539  return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
540  Advice);
541 }
542 
543 static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
544  switch (LTOPhase) {
546  return "main";
549  return "prelink";
552  return "postlink";
553  }
554  llvm_unreachable("unreachable");
555 }
556 
557 static inline const char *getInlineAdvisorContext(InlinePass IP) {
558  switch (IP) {
560  return "always-inline";
562  return "cgscc-inline";
564  return "early-inline";
565  case (InlinePass::MLInliner):
566  return "ml-inline";
568  return "module-inline";
570  return "replay-cgscc-inline";
572  return "replay-sample-profile-inline";
574  return "sample-profile-inline";
575  }
576 
577  llvm_unreachable("unreachable");
578 }
579 
581  return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
582  std::string(getInlineAdvisorContext(IC.Pass));
583 }
584 
588  auto &Callee = *CB.getCalledFunction();
589 
590  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
592  };
593 
594  auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
595 
596  auto TrivialDecision =
598 
599  if (TrivialDecision) {
600  if (TrivialDecision->isSuccess())
602  else
604  }
606 }
607 
608 std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
609  bool MandatoryOnly) {
610  if (!MandatoryOnly)
611  return getAdviceImpl(CB);
612  bool Advice = CB.getCaller() != CB.getCalledFunction() &&
615  return getMandatoryAdvice(CB, Advice);
616 }
617 
620 }
621 
624  const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
625  if (!IA)
626  OS << "No Inline Advisor\n";
627  else
628  IA->getAdvisor()->print(OS);
629  return PreservedAnalyses::all();
630 }
631 
634  CGSCCUpdateResult &UR) {
635  const auto &MAMProxy =
636  AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG);
637 
638  if (InitialC.size() == 0) {
639  OS << "SCC is empty!\n";
640  return PreservedAnalyses::all();
641  }
642  Module &M = *InitialC.begin()->getFunction().getParent();
643  const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
644  if (!IA)
645  OS << "No Inline Advisor\n";
646  else
647  IA->getAdvisor()->print(OS);
648  return PreservedAnalyses::all();
649 }
llvm::InlinePass::AlwaysInliner
@ AlwaysInliner
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::InlineCost::isAlways
bool isAlways() const
Definition: InlineCost.h:139
llvm::OuterAnalysisManagerProxy
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:1056
llvm::InlineCost::getCost
int getCost() const
Get the inline cost estimate.
Definition: InlineCost.h:145
AssumptionCache.h
llvm::TargetIRAnalysis
Analysis pass providing the TargetTransformInfo.
Definition: TargetTransformInfo.h:2592
llvm::OptimizationRemarkMissed
Diagnostic information for missed-optimization remarks.
Definition: DiagnosticInfo.h:734
llvm::InlineAdvice::recordInlining
void recordInlining()
Exactly one of the record* APIs must be called.
Definition: InlineAdvisor.cpp:186
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::getReleaseModeAdvisor
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM)
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::InlineAdvisor::ImportedFunctionsStats
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
Definition: InlineAdvisor.h:212
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:104
llvm::InlineAdvisor::M
Module & M
Definition: InlineAdvisor.h:208
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
DebugInfoMetadata.h
llvm::ThinOrFullLTOPhase::ThinLTOPostLink
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
llvm::DiagnosticHandler::isMissedOptRemarkEnabled
virtual bool isMissedOptRemarkEnabled(StringRef PassName) const
Return true if missed optimization remarks are enabled, override to provide different implementation.
Definition: DiagnosticHandler.cpp:75
llvm::Function
Definition: Function.h:60
llvm::Attribute
Definition: Attributes.h:67
llvm::InlinePass::EarlyInliner
@ EarlyInliner
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:628
llvm::InlinerFunctionImportStatsOpts::Verbose
@ Verbose
llvm::Attribute::get
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:91
Statistic.h
llvm::InlineAdvisorAnalysisPrinterPass::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
Definition: InlineAdvisor.cpp:623
llvm::InlineAdvice::Caller
Function *const Caller
Caller and Callee are pre-inlining.
Definition: InlineAdvisor.h:122
OptimizationRemarkEmitter.h
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
llvm::emitInlinedIntoBasedOnCost
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
Definition: InlineAdvisor.cpp:502
llvm::DILocation
Debug location.
Definition: DebugInfoMetadata.h:1600
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:321
llvm::Optional< llvm::InlineCost >
llvm::InlineParams
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:206
llvm::InlineAdvisor::MandatoryInliningKind::Always
@ Always
llvm::InlineAdvisor::MandatoryInliningKind::Never
@ Never
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::InlineCost::getThreshold
int getThreshold() const
Get the threshold against which the cost was computed.
Definition: InlineCost.h:151
llvm::ThinOrFullLTOPhase::ThinLTOPreLink
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
llvm::InlineCost::isNever
bool isNever() const
Definition: InlineCost.h:140
llvm::addLocationToRemarks
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
Definition: InlineAdvisor.cpp:458
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::InlineAdvisor::getCallerORE
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
Definition: InlineAdvisor.cpp:618
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
llvm::ThinOrFullLTOPhase
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:73
CommandLine.h
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
ImportedFunctionsInliningStatistics.h
llvm::InlinerFunctionImportStatsOpts::No
@ No
llvm::emitInlinedInto
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
Definition: InlineAdvisor.cpp:484
llvm::InlineAdvice::IsInliningRecommended
const bool IsInliningRecommended
Definition: InlineAdvisor.h:131
llvm::LazyCallGraph::SCC
An SCC of the call graph.
Definition: LazyCallGraph.h:419
InlineRemarkAttribute
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
llvm::getInlineCost
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
Definition: InlineCost.cpp:2822
llvm::InlineAdvice::ORE
OptimizationRemarkEmitter & ORE
Definition: InlineAdvisor.h:130
llvm::User
Definition: User.h:44
llvm::DefaultInlineAdvisor
The default (manual heuristics) implementation of the InlineAdvisor.
Definition: InlineAdvisor.h:229
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1396
MAM
ModuleAnalysisManager MAM
Definition: PassBuilderBindings.cpp:61
llvm::InlineCost
Represents the cost of inlining a function.
Definition: InlineCost.h:90
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
IP
Definition: NVPTXLowerArgs.cpp:168
TargetLibraryInfo.h
getLTOPhase
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
Definition: InlineAdvisor.cpp:543
First
into llvm powi allowing the code generator to produce balanced multiplication trees First
Definition: README.txt:54
InlineAdvisor.h
getInlineAdvisorContext
static const char * getInlineAdvisorContext(InlinePass IP)
Definition: InlineAdvisor.cpp:557
llvm::Instruction
Definition: Instruction.h:42
llvm::CallBase::addFnAttr
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Definition: InstrTypes.h:1506
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
llvm::operator<<
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:292
llvm::AArch64PACKey::IA
@ IA
Definition: AArch64BaseInfo.h:819
llvm::InlineContext
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:58
llvm::BlockFrequencyAnalysis
Analysis pass which computes BlockFrequencyInfo.
Definition: BlockFrequencyInfo.h:112
llvm::CallBase::getCaller
Function * getCaller()
Helper to get the caller (the parent function).
Definition: Instructions.cpp:285
llvm::formatCallSiteLocation
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
Definition: InlineAdvisor.cpp:430
llvm::ReplayInlinerSettings::ReplayFile
StringRef ReplayFile
Definition: ReplayInlineAdvisor.h:47
llvm::ProfileSummaryInfo
Analysis providing profile information.
Definition: ProfileSummaryInfo.h:40
llvm::InlineAdvisor::MandatoryInliningKind
MandatoryInliningKind
Definition: InlineAdvisor.h:214
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::StringRef::empty
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
llvm::LLVMContext::getDiagHandlerPtr
const DiagnosticHandler * getDiagHandlerPtr() const
getDiagHandlerPtr - Returns const raw pointer of DiagnosticHandler set by setDiagnosticHandler.
Definition: LLVMContext.cpp:363
llvm::HighlightColor::Remark
@ Remark
llvm::InlineAdvice
Capture state between an inlining decision having had been made, and its impact being observable.
Definition: InlineAdvisor.h:74
llvm::cl::opt< bool >
ReplayInlineAdvisor.h
llvm::InliningAdvisorMode
InliningAdvisorMode
There are 3 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:42
shouldBeDeferred
static bool shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
Definition: InlineAdvisor.cpp:243
llvm::DiagnosticInfoOptimizationBase::Argument
Used in the streaming interface as the general argument type.
Definition: DiagnosticInfo.h:426
llvm::InlineAdvisor::getAnnotatedInlinePassName
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
Definition: InlineAdvisor.h:197
llvm::InlineAdvisor::InlineAdvisor
InlineAdvisor(InlineAdvisor &&)=delete
ProfileSummaryInfo.h
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
llvm::InlineAdvice::recordInliningWithCalleeDeleted
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
Definition: InlineAdvisor.cpp:192
llvm::AssumptionAnalysis
A function analysis which provides an AssumptionCache.
Definition: AssumptionCache.h:173
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::InlinerFunctionImportStats
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
Definition: InlineAdvisor.cpp:65
llvm::AnalysisKey
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:69
llvm::InlinePass::MLInliner
@ MLInliner
llvm::DebugLoc::get
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
getCalledFunction
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
Definition: MemoryBuiltins.cpp:155
llvm::InlineAdvice::Block
const BasicBlock *const Block
Definition: InlineAdvisor.h:129
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:446
llvm::InlineCost::getCostDelta
int getCostDelta() const
Get the cost delta from the threshold for inlining.
Definition: InlineCost.h:175
llvm::InlineAdvice::recordInliningImpl
virtual void recordInliningImpl()
Definition: InlineAdvisor.h:115
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::setInlineRemark
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
Definition: InlineAdvisor.cpp:357
llvm::OptimizationRemarkEmitter::emit
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Definition: OptimizationRemarkEmitter.cpp:77
llvm::InlineAdvice::DLoc
const DebugLoc DLoc
Definition: InlineAdvisor.h:128
InlineCost.h
llvm::InlinePass::CGSCCInliner
@ CGSCCInliner
Mode
SI Whole Quad Mode
Definition: SIWholeQuadMode.cpp:264
llvm::InliningAdvisorMode::Development
@ Development
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::InlineContext::Pass
InlinePass Pass
Definition: InlineAdvisor.h:61
llvm::InlineParams::EnableDeferral
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.
Definition: InlineCost.h:236
getDebugLoc
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
Definition: MachineInstrBundle.cpp:110
llvm::ThinOrFullLTOPhase::FullLTOPostLink
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
llvm::InlineAdvisor::MandatoryInliningKind::NotMandatory
@ NotMandatory
llvm::InlineAdvisorAnalysis::Result::tryCreate
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
Definition: InlineAdvisor.cpp:200
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::CallSiteFormat
Definition: ReplayInlineAdvisor.h:21
llvm::InlineCost::getReason
const char * getReason() const
Get the reason of Always or Never.
Definition: InlineCost.h:166
llvm::AssumptionCache
A cache of @llvm.assume calls within a function.
Definition: AssumptionCache.h:42
llvm::InlineAdvisor
Interface for deciding whether to inline a call site or not.
Definition: InlineAdvisor.h:163
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
getParent
static const Function * getParent(const Value *V)
Definition: BasicAliasAnalysis.cpp:805
uint32_t
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
llvm::InlineAdvice::recordInliningWithCalleeDeletedImpl
virtual void recordInliningWithCalleeDeletedImpl()
Definition: InlineAdvisor.h:116
llvm::InlinePass::ReplaySampleProfileInliner
@ ReplaySampleProfileInliner
InlineDeferralScale
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:187
EnableInlineDeferral
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
llvm::InlineAdvisor::getMandatoryAdvice
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
Definition: InlineAdvisor.cpp:537
llvm::getAttributeBasedInliningDecision
Optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
Definition: InlineCost.cpp:2870
llvm::CGSCCUpdateResult
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Definition: CGSCCPassManager.h:232
llvm::InlineAdvisor::FAM
FunctionAnalysisManager & FAM
Definition: InlineAdvisor.h:209
llvm::InlinePass::ModuleInliner
@ ModuleInliner
llvm::InlineAdvisor::getAdvice
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
Definition: InlineAdvisor.cpp:608
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::InlinePass
InlinePass
Definition: InlineAdvisor.h:45
llvm::AnalysisManager::getCachedResult
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:793
llvm::Module::getContext
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:262
AnnotateInlinePhase
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
llvm::InlineAdvisor::getAdviceImpl
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
PassManager.h
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:226
llvm::ThinOrFullLTOPhase::None
@ None
No LTO/ThinLTO behavior needed.
llvm::inlineCostStr
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
Definition: InlineAdvisor.cpp:350
llvm::InlinePass::SampleProfileInliner
@ SampleProfileInliner
llvm::InlineAdvisor::getMandatoryKind
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
Definition: InlineAdvisor.cpp:586
llvm::AnnotateInlinePassName
std::string AnnotateInlinePassName(InlineContext IC)
Definition: InlineAdvisor.cpp:580
llvm::ThinOrFullLTOPhase::FullLTOPreLink
@ FullLTOPreLink
Full LTO prelink phase.
llvm::OptimizationRemark
Diagnostic information for applied optimization remarks.
Definition: DiagnosticInfo.h:689
llvm::InlineAdvisorAnalysis
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Definition: InlineAdvisor.h:243
llvm::InlineConstants::LastCallToStaticBonus
const int LastCallToStaticBonus
Definition: InlineCost.h:51
llvm::InlineAdvisorAnalysis::Key
static AnalysisKey Key
Definition: InlineAdvisor.h:245
llvm::InliningAdvisorMode::Default
@ Default
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::getDevelopmentModeAdvisor
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
TargetTransformInfo.h
llvm::InlineAdvice::InlineAdvice
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
Definition: InlineAdvisor.cpp:174
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1174
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::InnerAnalysisManagerProxy
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:931
llvm::InliningAdvisorMode::Release
@ Release
DEBUG_TYPE
#define DEBUG_TYPE
Definition: InlineAdvisor.cpp:30
llvm::ore
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
Definition: OptimizationRemarkEmitter.h:135
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:412
llvm::LazyCallGraph
A lazily constructed view of the call graph of a module.
Definition: LazyCallGraph.h:110
raw_ostream.h
getDefaultInlineAdvice
static llvm::Optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
Definition: InlineAdvisor.cpp:133
llvm::raw_string_ostream::str
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:646
llvm::InlineAdvice::Advisor
InlineAdvisor *const Advisor
Definition: InlineAdvisor.h:120
llvm::OptimizationRemarkEmitterAnalysis
Definition: OptimizationRemarkEmitter.h:164
llvm::InlineResult
InlineResult is basically true or false.
Definition: InlineCost.h:180
llvm::InlineAdvice::Callee
Function *const Callee
Definition: InlineAdvisor.h:123
llvm::TargetLibraryAnalysis
Analysis pass providing the TargetLibraryInfo.
Definition: TargetLibraryInfo.h:450
llvm::ReplayInlinerSettings
Replay Inliner Setup.
Definition: ReplayInlineAdvisor.h:43
llvm::InlineAdvisor::~InlineAdvisor
virtual ~InlineAdvisor()
Definition: InlineAdvisor.cpp:529
PassName
static const char PassName[]
Definition: X86LowerAMXIntrinsics.cpp:671
llvm::InlinePass::ReplayCGSCCInliner
@ ReplayCGSCCInliner
llvm::getReplayInlineAdvisor
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
Definition: ReplayInlineAdvisor.cpp:80
llvm::shouldInline
Optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
Definition: InlineAdvisor.cpp:370
llvm::InlineContext::LTOPhase
ThinOrFullLTOPhase LTOPhase
Definition: InlineAdvisor.h:59