LLVM 17.0.0git
InlineAdvisor.cpp
Go to the documentation of this file.
1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10// related types.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/Statistic.h"
25#include "llvm/IR/PassManager.h"
28
29using namespace llvm;
30#define DEBUG_TYPE "inline"
31#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
32#define LLVM_HAVE_TF_AOT
33#endif
34
35// This weirdly named statistic tracks the number of times that, when attempting
36// to inline a function A into B, we analyze the callers of B in order to see
37// if those would be more profitable and blocked inline steps.
38STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
39
40/// Flag to add inline messages as callsite attributes 'inline-remark'.
41static cl::opt<bool>
42 InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
44 cl::desc("Enable adding inline-remark attribute to"
45 " callsites processed by inliner but decided"
46 " to be not inlined"));
47
48static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
50 cl::desc("Enable deferred inlining"));
51
52// An integer used to limit the cost of inline deferral. The default negative
53// number tells shouldBeDeferred to only take the secondary cost into account.
54static cl::opt<int>
55 InlineDeferralScale("inline-deferral-scale",
56 cl::desc("Scale to limit the cost of inline deferral"),
58
59static cl::opt<bool>
60 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
61 cl::desc("If true, annotate inline advisor remarks "
62 "with LTO and pass information."));
63
64namespace llvm {
66}
67
68namespace {
69using namespace llvm::ore;
70class MandatoryInlineAdvice : public InlineAdvice {
71public:
72 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
74 bool IsInliningMandatory)
75 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
76
77private:
79
80 void recordInliningImpl() override {
81 if (IsInliningRecommended)
82 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
84 Remark << ": always inline attribute";
85 });
86 }
87
88 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
89 if (IsInliningRecommended)
90 ORE.emit([&]() {
92 "NotInlined", DLoc, Block)
93 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
94 << NV("Caller", Caller)
95 << "': " << NV("Reason", Result.getFailureReason());
96 });
97 }
98
99 void recordUnattemptedInliningImpl() override {
100 assert(!IsInliningRecommended && "Expected to attempt inlining");
101 }
102};
103} // namespace
104
105void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
106 const InlineResult &Result) {
107 using namespace ore;
108 llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
109 "; " + inlineCostStr(*OIC));
110 ORE.emit([&]() {
112 "NotInlined", DLoc, Block)
113 << "'" << NV("Callee", Callee) << "' is not inlined into '"
114 << NV("Caller", Caller)
115 << "': " << NV("Reason", Result.getFailureReason());
116 });
117}
118
119void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
120 if (EmitRemarks)
122 /* ForProfileContext= */ false,
124}
125
126void DefaultInlineAdvice::recordInliningImpl() {
127 if (EmitRemarks)
129 /* ForProfileContext= */ false,
131}
132
133std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
134 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
135 Function &Caller = *CB.getCaller();
136 ProfileSummaryInfo *PSI =
138 .getCachedResult<ProfileSummaryAnalysis>(
139 *CB.getParent()->getParent()->getParent());
140
142 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
144 };
145 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
147 };
148 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
150 };
151
152 auto GetInlineCost = [&](CallBase &CB) {
154 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
155 bool RemarksEnabled =
156 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
157 DEBUG_TYPE);
158 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
159 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
160 };
161 return llvm::shouldInline(
162 CB, GetInlineCost, ORE,
163 Params.EnableDeferral.value_or(EnableInlineDeferral));
164}
165
166std::unique_ptr<InlineAdvice>
167DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
168 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
169 return std::make_unique<DefaultInlineAdvice>(
170 this, CB, OIC,
172}
173
176 bool IsInliningRecommended)
177 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
178 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
179 IsInliningRecommended(IsInliningRecommended) {}
180
181void InlineAdvice::recordInlineStatsIfNeeded() {
183 Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
184}
185
187 markRecorded();
188 recordInlineStatsIfNeeded();
190}
191
193 markRecorded();
194 recordInlineStatsIfNeeded();
196}
197
201
204 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
205 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
207 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M);
208 Advisor.reset(DA.Factory(M, FAM, Params, IC));
209 return !!Advisor;
210 }
211 switch (Mode) {
213 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
214 Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
215 // Restrict replay to default advisor, ML advisors are stateful so
216 // replay will need augmentations to interleave with them correctly.
217 if (!ReplaySettings.ReplayFile.empty()) {
219 std::move(Advisor), ReplaySettings,
220 /* EmitRemarks =*/true, IC);
221 }
222 break;
224#ifdef LLVM_HAVE_TFLITE
225 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
226 Advisor =
227 llvm::getDevelopmentModeAdvisor(M, MAM, [&FAM, Params](CallBase &CB) {
228 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
229 return OIC.has_value();
230 });
231#endif
232 break;
234 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
235 Advisor = llvm::getReleaseModeAdvisor(M, MAM);
236 break;
237 }
238
239 return !!Advisor;
240}
241
242/// Return true if inlining of CB can block the caller from being
243/// inlined which is proved to be more beneficial. \p IC is the
244/// estimated inline cost associated with callsite \p CB.
245/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
246/// caller if \p CB is suppressed for inlining.
247static bool
248shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
249 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
250 // For now we only handle local or inline functions.
251 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
252 return false;
253 // If the cost of inlining CB is non-positive, it is not going to prevent the
254 // caller from being inlined into its callers and hence we don't need to
255 // defer.
256 if (IC.getCost() <= 0)
257 return false;
258 // Try to detect the case where the current inlining candidate caller (call
259 // it B) is a static or linkonce-ODR function and is an inlining candidate
260 // elsewhere, and the current candidate callee (call it C) is large enough
261 // that inlining it into B would make B too big to inline later. In these
262 // circumstances it may be best not to inline C into B, but to inline B into
263 // its callers.
264 //
265 // This only applies to static and linkonce-ODR functions because those are
266 // expected to be available for inlining in the translation units where they
267 // are used. Thus we will always have the opportunity to make local inlining
268 // decisions. Importantly the linkonce-ODR linkage covers inline functions
269 // and templates in C++.
270 //
271 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
272 // the internal implementation of the inline cost metrics rather than
273 // treating them as truly abstract units etc.
274 TotalSecondaryCost = 0;
275 // The candidate cost to be imposed upon the current function.
276 int CandidateCost = IC.getCost() - 1;
277 // If the caller has local linkage and can be inlined to all its callers, we
278 // can apply a huge negative bonus to TotalSecondaryCost.
279 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
280 // This bool tracks what happens if we DO inline C into B.
281 bool InliningPreventsSomeOuterInline = false;
282 unsigned NumCallerUsers = 0;
283 for (User *U : Caller->users()) {
284 CallBase *CS2 = dyn_cast<CallBase>(U);
285
286 // If this isn't a call to Caller (it could be some other sort
287 // of reference) skip it. Such references will prevent the caller
288 // from being removed.
289 if (!CS2 || CS2->getCalledFunction() != Caller) {
290 ApplyLastCallBonus = false;
291 continue;
292 }
293
294 InlineCost IC2 = GetInlineCost(*CS2);
295 ++NumCallerCallersAnalyzed;
296 if (!IC2) {
297 ApplyLastCallBonus = false;
298 continue;
299 }
300 if (IC2.isAlways())
301 continue;
302
303 // See if inlining of the original callsite would erase the cost delta of
304 // this callsite. We subtract off the penalty for the call instruction,
305 // which we would be deleting.
306 if (IC2.getCostDelta() <= CandidateCost) {
307 InliningPreventsSomeOuterInline = true;
308 TotalSecondaryCost += IC2.getCost();
309 NumCallerUsers++;
310 }
311 }
312
313 if (!InliningPreventsSomeOuterInline)
314 return false;
315
316 // If all outer calls to Caller would get inlined, the cost for the last
317 // one is set very low by getInlineCost, in anticipation that Caller will
318 // be removed entirely. We did not account for this above unless there
319 // is only one caller of Caller.
320 if (ApplyLastCallBonus)
321 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
322
323 // If InlineDeferralScale is negative, then ignore the cost of primary
324 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
325 if (InlineDeferralScale < 0)
326 return TotalSecondaryCost < IC.getCost();
327
328 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
329 int Allowance = IC.getCost() * InlineDeferralScale;
330 return TotalCost < Allowance;
331}
332
333namespace llvm {
335 return R << Arg.Val;
336}
337
338template <class RemarkT>
339RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
340 using namespace ore;
341 if (IC.isAlways()) {
342 R << "(cost=always)";
343 } else if (IC.isNever()) {
344 R << "(cost=never)";
345 } else {
346 R << "(cost=" << ore::NV("Cost", IC.getCost())
347 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
348 }
349 if (const char *Reason = IC.getReason())
350 R << ": " << ore::NV("Reason", Reason);
351 return R;
352}
353} // namespace llvm
354
355std::string llvm::inlineCostStr(const InlineCost &IC) {
356 std::string Buffer;
358 Remark << IC;
359 return Remark.str();
360}
361
364 return;
365
366 Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
367 CB.addFnAttr(Attr);
368}
369
370/// Return the cost only if the inliner should attempt to inline at the given
371/// CallSite. If we return the cost, we will emit an optimisation remark later
372/// using that cost, so we won't do so from this function. Return std::nullopt
373/// if inlining should not be attempted.
374std::optional<InlineCost>
376 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
377 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
378 using namespace ore;
379
380 InlineCost IC = GetInlineCost(CB);
381 Instruction *Call = &CB;
383 Function *Caller = CB.getCaller();
384
385 if (IC.isAlways()) {
386 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
387 << ", Call: " << CB << "\n");
388 return IC;
389 }
390
391 if (!IC) {
392 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
393 << ", Call: " << CB << "\n");
394 if (IC.isNever()) {
395 ORE.emit([&]() {
396 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
397 << "'" << NV("Callee", Callee) << "' not inlined into '"
398 << NV("Caller", Caller)
399 << "' because it should never be inlined " << IC;
400 });
401 } else {
402 ORE.emit([&]() {
403 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
404 << "'" << NV("Callee", Callee) << "' not inlined into '"
405 << NV("Caller", Caller) << "' because too costly to inline "
406 << IC;
407 });
408 }
410 return std::nullopt;
411 }
412
413 int TotalSecondaryCost = 0;
414 if (EnableDeferral &&
415 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
416 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
417 << " Cost = " << IC.getCost()
418 << ", outer Cost = " << TotalSecondaryCost << '\n');
419 ORE.emit([&]() {
420 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
421 Call)
422 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
423 << "' increases the cost of inlining '" << NV("Caller", Caller)
424 << "' in other contexts";
425 });
426 setInlineRemark(CB, "deferred");
427 return std::nullopt;
428 }
429
430 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
431 << '\n');
432 return IC;
433}
434
436 const CallSiteFormat &Format) {
437 std::string Buffer;
438 raw_string_ostream CallSiteLoc(Buffer);
439 bool First = true;
440 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
441 if (!First)
442 CallSiteLoc << " @ ";
443 // Note that negative line offset is actually possible, but we use
444 // unsigned int to match line offset representation in remarks so
445 // it's directly consumable by relay advisor.
447 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
448 uint32_t Discriminator = DIL->getBaseDiscriminator();
449 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
450 if (Name.empty())
451 Name = DIL->getScope()->getSubprogram()->getName();
452 CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
453 if (Format.outputColumn())
454 CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
455 if (Format.outputDiscriminator() && Discriminator)
456 CallSiteLoc << "." << llvm::utostr(Discriminator);
457 First = false;
458 }
459
460 return CallSiteLoc.str();
461}
462
464 if (!DLoc) {
465 return;
466 }
467
468 bool First = true;
469 Remark << " at callsite ";
470 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
471 if (!First)
472 Remark << " @ ";
473 unsigned int Offset = DIL->getLine();
474 Offset -= DIL->getScope()->getSubprogram()->getLine();
475 unsigned int Discriminator = DIL->getBaseDiscriminator();
476 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
477 if (Name.empty())
478 Name = DIL->getScope()->getSubprogram()->getName();
479 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
480 << ore::NV("Column", DIL->getColumn());
481 if (Discriminator)
482 Remark << "." << ore::NV("Disc", Discriminator);
483 First = false;
484 }
485
486 Remark << ";";
487}
488
491 const Function &Callee, const Function &Caller, bool AlwaysInline,
492 function_ref<void(OptimizationRemark &)> ExtraContext,
493 const char *PassName) {
494 ORE.emit([&]() {
495 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
497 DLoc, Block);
498 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
499 << ore::NV("Caller", &Caller) << "'";
500 if (ExtraContext)
501 ExtraContext(Remark);
503 return Remark;
504 });
505}
506
509 const Function &Callee, const Function &Caller, const InlineCost &IC,
510 bool ForProfileContext, const char *PassName) {
512 ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
514 if (ForProfileContext)
515 Remark << " to match profiling context";
516 Remark << " with " << IC;
517 },
518 PassName);
519}
520
522 std::optional<InlineContext> IC)
523 : M(M), FAM(FAM), IC(IC),
524 AnnotatedInlinePassName((IC && AnnotateInlinePhase)
526 : DEBUG_TYPE) {
529 std::make_unique<ImportedFunctionsInliningStatistics>();
530 ImportedFunctionsStats->setModuleInfo(M);
531 }
532}
533
539 }
540}
541
542std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
543 bool Advice) {
544 return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
545 Advice);
546}
547
548static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
549 switch (LTOPhase) {
551 return "main";
554 return "prelink";
557 return "postlink";
558 }
559 llvm_unreachable("unreachable");
560}
561
562static inline const char *getInlineAdvisorContext(InlinePass IP) {
563 switch (IP) {
565 return "always-inline";
567 return "cgscc-inline";
569 return "early-inline";
571 return "ml-inline";
573 return "module-inline";
575 return "replay-cgscc-inline";
577 return "replay-sample-profile-inline";
579 return "sample-profile-inline";
580 }
581
582 llvm_unreachable("unreachable");
583}
584
586 return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
587 std::string(getInlineAdvisorContext(IC.Pass));
588}
589
593 auto &Callee = *CB.getCalledFunction();
594
595 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
597 };
598
599 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
600
601 auto TrivialDecision =
603
604 if (TrivialDecision) {
605 if (TrivialDecision->isSuccess())
607 else
609 }
611}
612
613std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
614 bool MandatoryOnly) {
615 if (!MandatoryOnly)
616 return getAdviceImpl(CB);
617 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
620 return getMandatoryAdvice(CB, Advice);
621}
622
625}
626
629 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
630 if (!IA)
631 OS << "No Inline Advisor\n";
632 else
633 IA->getAdvisor()->print(OS);
634 return PreservedAnalyses::all();
635}
636
639 CGSCCUpdateResult &UR) {
640 const auto &MAMProxy =
642
643 if (InitialC.size() == 0) {
644 OS << "SCC is empty!\n";
645 return PreservedAnalyses::all();
646 }
647 Module &M = *InitialC.begin()->getFunction().getParent();
648 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
649 if (!IA)
650 OS << "No Inline Advisor\n";
651 else
652 IA->getAdvisor()->print(OS);
653 return PreservedAnalyses::all();
654}
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static const Function * getParent(const Value *V)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
#define DEBUG_TYPE
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
static std::optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
static const char * getInlineAdvisorContext(InlinePass IP)
static bool shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
#define F(x, y, z)
Definition: MD5.cpp:55
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This pass exposes codegen information to IR-level passes.
static const char PassName[]
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:793
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:91
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1186
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Definition: InstrTypes.h:1518
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1408
Function * getCaller()
Helper to get the caller (the parent function).
Debug location.
A debug info location.
Definition: DebugLoc.h:33
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
The default (manual heuristics) implementation of the InlineAdvisor.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
Capture state between an inlining decision having had been made, and its impact being observable.
Definition: InlineAdvisor.h:76
Function *const Callee
Function *const Caller
Caller and Callee are pre-inlining.
virtual void recordInliningWithCalleeDeletedImpl()
virtual void recordInliningImpl()
const BasicBlock *const Block
void recordInlining()
Exactly one of the record* APIs must be called.
OptimizationRemarkEmitter & ORE
virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result)
InlineAdvisor *const Advisor
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
virtual void recordUnattemptedInliningImpl()
const DebugLoc DLoc
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
static AnalysisKey Key
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
InlineAdvisor(InlineAdvisor &&)=delete
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
Represents the cost of inlining a function.
Definition: InlineCost.h:89
int getThreshold() const
Get the threshold against which the cost was computed.
Definition: InlineCost.h:150
const char * getReason() const
Get the reason of Always or Never.
Definition: InlineCost.h:165
int getCost() const
Get the inline cost estimate.
Definition: InlineCost.h:144
bool isAlways() const
Definition: InlineCost.h:138
int getCostDelta() const
Get the cost delta from the threshold for inlining.
Definition: InlineCost.h:174
bool isNever() const
Definition: InlineCost.h:139
InlineResult is basically true or false.
Definition: InlineCost.h:179
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:933
const BasicBlock * getParent() const
Definition: Instruction.h:90
An SCC of the call graph.
iterator begin() const
A lazily constructed view of the call graph of a module.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:262
The optimization diagnostic interface.
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:1058
Used for dynamically registering InlineAdvisors as plugins.
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
Analysis providing profile information.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:660
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const int LastCallToStaticBonus
Definition: InlineCost.h:50
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:44
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:73
@ FullLTOPreLink
Full LTO prelink phase.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
@ None
No LTO/ThinLTO behavior needed.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
std::optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
std::optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:292
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM)
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:69
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Used in the streaming interface as the general argument type.
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:60
ThinOrFullLTOPhase LTOPhase
Definition: InlineAdvisor.h:61
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:205
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.
Definition: InlineCost.h:235