LLVM 20.0.0git
InlineAdvisor.cpp
Go to the documentation of this file.
1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10// related types.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/Statistic.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PassManager.h"
30
31using namespace llvm;
32#define DEBUG_TYPE "inline"
33#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
34#define LLVM_HAVE_TF_AOT
35#endif
36
37// This weirdly named statistic tracks the number of times that, when attempting
38// to inline a function A into B, we analyze the callers of B in order to see
39// if those would be more profitable and blocked inline steps.
40STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
41
42/// Flag to add inline messages as callsite attributes 'inline-remark'.
43static cl::opt<bool>
44 InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
46 cl::desc("Enable adding inline-remark attribute to"
47 " callsites processed by inliner but decided"
48 " to be not inlined"));
49
50static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
52 cl::desc("Enable deferred inlining"));
53
54// An integer used to limit the cost of inline deferral. The default negative
55// number tells shouldBeDeferred to only take the secondary cost into account.
56static cl::opt<int>
57 InlineDeferralScale("inline-deferral-scale",
58 cl::desc("Scale to limit the cost of inline deferral"),
60
61static cl::opt<bool>
62 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
63 cl::desc("If true, annotate inline advisor remarks "
64 "with LTO and pass information."));
65
66namespace llvm {
68} // namespace llvm
69
70namespace {
71using namespace llvm::ore;
72class MandatoryInlineAdvice : public InlineAdvice {
73public:
74 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
76 bool IsInliningMandatory)
77 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
78
79private:
81
82 void recordInliningImpl() override {
83 if (IsInliningRecommended)
84 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
86 Remark << ": always inline attribute";
87 });
88 }
89
90 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
91 if (IsInliningRecommended)
92 ORE.emit([&]() {
94 "NotInlined", DLoc, Block)
95 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
96 << NV("Caller", Caller)
97 << "': " << NV("Reason", Result.getFailureReason());
98 });
99 }
100
101 void recordUnattemptedInliningImpl() override {
102 assert(!IsInliningRecommended && "Expected to attempt inlining");
103 }
104};
105} // namespace
106
107void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
108 const InlineResult &Result) {
109 using namespace ore;
110 llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
111 "; " + inlineCostStr(*OIC));
112 ORE.emit([&]() {
114 "NotInlined", DLoc, Block)
115 << "'" << NV("Callee", Callee) << "' is not inlined into '"
116 << NV("Caller", Caller)
117 << "': " << NV("Reason", Result.getFailureReason());
118 });
119}
120
121void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
122 if (EmitRemarks)
124 /* ForProfileContext= */ false,
126}
127
128void DefaultInlineAdvice::recordInliningImpl() {
129 if (EmitRemarks)
131 /* ForProfileContext= */ false,
133}
134
135std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
136 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
137 Function &Caller = *CB.getCaller();
138 ProfileSummaryInfo *PSI =
140 .getCachedResult<ProfileSummaryAnalysis>(
141 *CB.getParent()->getParent()->getParent());
142
144 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
146 };
147 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
149 };
150 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
152 };
153
154 auto GetInlineCost = [&](CallBase &CB) {
155 Function &Callee = *CB.getCalledFunction();
156 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
157 bool RemarksEnabled =
158 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
159 DEBUG_TYPE);
160 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
161 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
162 };
163 return llvm::shouldInline(
164 CB, GetInlineCost, ORE,
165 Params.EnableDeferral.value_or(EnableInlineDeferral));
166}
167
168std::unique_ptr<InlineAdvice>
169DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
170 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
171 return std::make_unique<DefaultInlineAdvice>(
172 this, CB, OIC,
174}
175
178 bool IsInliningRecommended)
179 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
180 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
181 IsInliningRecommended(IsInliningRecommended) {}
182
183void InlineAdvice::recordInlineStatsIfNeeded() {
185 Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
186}
187
189 markRecorded();
190 recordInlineStatsIfNeeded();
192}
193
195 markRecorded();
196 recordInlineStatsIfNeeded();
198}
199
203
206 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
207 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
209 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M);
210 Advisor.reset(DA.Factory(M, FAM, Params, IC));
211 return !!Advisor;
212 }
213 auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
214 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
215 return OIC.has_value();
216 };
217 switch (Mode) {
219 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
220 Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
221 // Restrict replay to default advisor, ML advisors are stateful so
222 // replay will need augmentations to interleave with them correctly.
223 if (!ReplaySettings.ReplayFile.empty()) {
225 std::move(Advisor), ReplaySettings,
226 /* EmitRemarks =*/true, IC);
227 }
228 break;
230#ifdef LLVM_HAVE_TFLITE
231 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
232 Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
233#endif
234 break;
236 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
237 Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
238 break;
239 }
240
241 return !!Advisor;
242}
243
244/// Return true if inlining of CB can block the caller from being
245/// inlined which is proved to be more beneficial. \p IC is the
246/// estimated inline cost associated with callsite \p CB.
247/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
248/// caller if \p CB is suppressed for inlining.
249static bool
250shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
251 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
252 // For now we only handle local or inline functions.
253 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
254 return false;
255 // If the cost of inlining CB is non-positive, it is not going to prevent the
256 // caller from being inlined into its callers and hence we don't need to
257 // defer.
258 if (IC.getCost() <= 0)
259 return false;
260 // Try to detect the case where the current inlining candidate caller (call
261 // it B) is a static or linkonce-ODR function and is an inlining candidate
262 // elsewhere, and the current candidate callee (call it C) is large enough
263 // that inlining it into B would make B too big to inline later. In these
264 // circumstances it may be best not to inline C into B, but to inline B into
265 // its callers.
266 //
267 // This only applies to static and linkonce-ODR functions because those are
268 // expected to be available for inlining in the translation units where they
269 // are used. Thus we will always have the opportunity to make local inlining
270 // decisions. Importantly the linkonce-ODR linkage covers inline functions
271 // and templates in C++.
272 //
273 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
274 // the internal implementation of the inline cost metrics rather than
275 // treating them as truly abstract units etc.
276 TotalSecondaryCost = 0;
277 // The candidate cost to be imposed upon the current function.
278 int CandidateCost = IC.getCost() - 1;
279 // If the caller has local linkage and can be inlined to all its callers, we
280 // can apply a huge negative bonus to TotalSecondaryCost.
281 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
282 // This bool tracks what happens if we DO inline C into B.
283 bool InliningPreventsSomeOuterInline = false;
284 unsigned NumCallerUsers = 0;
285 for (User *U : Caller->users()) {
286 CallBase *CS2 = dyn_cast<CallBase>(U);
287
288 // If this isn't a call to Caller (it could be some other sort
289 // of reference) skip it. Such references will prevent the caller
290 // from being removed.
291 if (!CS2 || CS2->getCalledFunction() != Caller) {
292 ApplyLastCallBonus = false;
293 continue;
294 }
295
296 InlineCost IC2 = GetInlineCost(*CS2);
297 ++NumCallerCallersAnalyzed;
298 if (!IC2) {
299 ApplyLastCallBonus = false;
300 continue;
301 }
302 if (IC2.isAlways())
303 continue;
304
305 // See if inlining of the original callsite would erase the cost delta of
306 // this callsite. We subtract off the penalty for the call instruction,
307 // which we would be deleting.
308 if (IC2.getCostDelta() <= CandidateCost) {
309 InliningPreventsSomeOuterInline = true;
310 TotalSecondaryCost += IC2.getCost();
311 NumCallerUsers++;
312 }
313 }
314
315 if (!InliningPreventsSomeOuterInline)
316 return false;
317
318 // If all outer calls to Caller would get inlined, the cost for the last
319 // one is set very low by getInlineCost, in anticipation that Caller will
320 // be removed entirely. We did not account for this above unless there
321 // is only one caller of Caller.
322 if (ApplyLastCallBonus)
323 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
324
325 // If InlineDeferralScale is negative, then ignore the cost of primary
326 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
327 if (InlineDeferralScale < 0)
328 return TotalSecondaryCost < IC.getCost();
329
330 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
331 int Allowance = IC.getCost() * InlineDeferralScale;
332 return TotalCost < Allowance;
333}
334
335namespace llvm {
336static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
337 return R << Arg.Val;
338}
339
340template <class RemarkT>
341RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
342 using namespace ore;
343 if (IC.isAlways()) {
344 R << "(cost=always)";
345 } else if (IC.isNever()) {
346 R << "(cost=never)";
347 } else {
348 R << "(cost=" << ore::NV("Cost", IC.getCost())
349 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
350 }
351 if (const char *Reason = IC.getReason())
352 R << ": " << ore::NV("Reason", Reason);
353 return R;
354}
355} // namespace llvm
356
357std::string llvm::inlineCostStr(const InlineCost &IC) {
358 std::string Buffer;
360 Remark << IC;
361 return Remark.str();
362}
363
366 return;
367
368 Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
369 CB.addFnAttr(Attr);
370}
371
372/// Return the cost only if the inliner should attempt to inline at the given
373/// CallSite. If we return the cost, we will emit an optimisation remark later
374/// using that cost, so we won't do so from this function. Return std::nullopt
375/// if inlining should not be attempted.
376std::optional<InlineCost>
378 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
379 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
380 using namespace ore;
381
382 InlineCost IC = GetInlineCost(CB);
383 Instruction *Call = &CB;
384 Function *Callee = CB.getCalledFunction();
385 Function *Caller = CB.getCaller();
386
387 if (IC.isAlways()) {
388 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
389 << ", Call: " << CB << "\n");
390 return IC;
391 }
392
393 if (!IC) {
394 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
395 << ", Call: " << CB << "\n");
396 if (IC.isNever()) {
397 ORE.emit([&]() {
398 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
399 << "'" << NV("Callee", Callee) << "' not inlined into '"
400 << NV("Caller", Caller)
401 << "' because it should never be inlined " << IC;
402 });
403 } else {
404 ORE.emit([&]() {
405 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
406 << "'" << NV("Callee", Callee) << "' not inlined into '"
407 << NV("Caller", Caller) << "' because too costly to inline "
408 << IC;
409 });
410 }
412 return std::nullopt;
413 }
414
415 int TotalSecondaryCost = 0;
416 if (EnableDeferral &&
417 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
418 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
419 << " Cost = " << IC.getCost()
420 << ", outer Cost = " << TotalSecondaryCost << '\n');
421 ORE.emit([&]() {
422 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
423 Call)
424 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
425 << "' increases the cost of inlining '" << NV("Caller", Caller)
426 << "' in other contexts";
427 });
428 setInlineRemark(CB, "deferred");
429 return std::nullopt;
430 }
431
432 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
433 << '\n');
434 return IC;
435}
436
438 const CallSiteFormat &Format) {
439 std::string Buffer;
440 raw_string_ostream CallSiteLoc(Buffer);
441 bool First = true;
442 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
443 if (!First)
444 CallSiteLoc << " @ ";
445 // Note that negative line offset is actually possible, but we use
446 // unsigned int to match line offset representation in remarks so
447 // it's directly consumable by relay advisor.
449 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
450 uint32_t Discriminator = DIL->getBaseDiscriminator();
451 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
452 if (Name.empty())
453 Name = DIL->getScope()->getSubprogram()->getName();
454 CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
455 if (Format.outputColumn())
456 CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
457 if (Format.outputDiscriminator() && Discriminator)
458 CallSiteLoc << "." << llvm::utostr(Discriminator);
459 First = false;
460 }
461
462 return CallSiteLoc.str();
463}
464
466 if (!DLoc) {
467 return;
468 }
469
470 bool First = true;
471 Remark << " at callsite ";
472 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
473 if (!First)
474 Remark << " @ ";
475 unsigned int Offset = DIL->getLine();
476 Offset -= DIL->getScope()->getSubprogram()->getLine();
477 unsigned int Discriminator = DIL->getBaseDiscriminator();
478 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
479 if (Name.empty())
480 Name = DIL->getScope()->getSubprogram()->getName();
481 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
482 << ore::NV("Column", DIL->getColumn());
483 if (Discriminator)
484 Remark << "." << ore::NV("Disc", Discriminator);
485 First = false;
486 }
487
488 Remark << ";";
489}
490
493 const Function &Callee, const Function &Caller, bool AlwaysInline,
494 function_ref<void(OptimizationRemark &)> ExtraContext,
495 const char *PassName) {
496 ORE.emit([&]() {
497 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
499 DLoc, Block);
500 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
501 << ore::NV("Caller", &Caller) << "'";
502 if (ExtraContext)
503 ExtraContext(Remark);
505 return Remark;
506 });
507}
508
511 const Function &Callee, const Function &Caller, const InlineCost &IC,
512 bool ForProfileContext, const char *PassName) {
514 ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
516 if (ForProfileContext)
517 Remark << " to match profiling context";
518 Remark << " with " << IC;
519 },
520 PassName);
521}
522
524 std::optional<InlineContext> IC)
525 : M(M), FAM(FAM), IC(IC),
526 AnnotatedInlinePassName((IC && AnnotateInlinePhase)
528 : DEBUG_TYPE) {
531 std::make_unique<ImportedFunctionsInliningStatistics>();
532 ImportedFunctionsStats->setModuleInfo(M);
533 }
534}
535
541 }
542}
543
544std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
545 bool Advice) {
546 return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
547 Advice);
548}
549
550static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
551 switch (LTOPhase) {
553 return "main";
556 return "prelink";
559 return "postlink";
560 }
561 llvm_unreachable("unreachable");
562}
563
564static inline const char *getInlineAdvisorContext(InlinePass IP) {
565 switch (IP) {
567 return "always-inline";
569 return "cgscc-inline";
571 return "early-inline";
573 return "ml-inline";
575 return "module-inline";
577 return "replay-cgscc-inline";
579 return "replay-sample-profile-inline";
581 return "sample-profile-inline";
582 }
583
584 llvm_unreachable("unreachable");
585}
586
588 return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
589 std::string(getInlineAdvisorContext(IC.Pass));
590}
591
595 auto &Callee = *CB.getCalledFunction();
596
597 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
599 };
600
601 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
602
603 auto TrivialDecision =
604 llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
605
606 if (TrivialDecision) {
607 if (TrivialDecision->isSuccess())
609 else
611 }
613}
614
615std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
616 bool MandatoryOnly) {
617 if (!MandatoryOnly)
618 return getAdviceImpl(CB);
619 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
622 return getMandatoryAdvice(CB, Advice);
623}
624
627}
628
631 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
632 if (!IA)
633 OS << "No Inline Advisor\n";
634 else
635 IA->getAdvisor()->print(OS);
636 return PreservedAnalyses::all();
637}
638
641 CGSCCUpdateResult &UR) {
642 const auto &MAMProxy =
644
645 if (InitialC.size() == 0) {
646 OS << "SCC is empty!\n";
647 return PreservedAnalyses::all();
648 }
649 Module &M = *InitialC.begin()->getFunction().getParent();
650 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
651 if (!IA)
652 OS << "No Inline Advisor\n";
653 else
654 IA->getAdvisor()->print(OS);
655 return PreservedAnalyses::all();
656}
static const Function * getParent(const Value *V)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
#define DEBUG_TYPE
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
static std::optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
static const char * getInlineAdvisorContext(InlinePass IP)
static bool shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
#define F(x, y, z)
Definition: MD5.cpp:55
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
static const Function * getCalledFunction(const Value *V)
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file contains some functions that are useful when dealing with strings.
This pass exposes codegen information to IR-level passes.
static const char PassName[]
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:424
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:94
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Definition: InstrTypes.h:1574
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
Function * getCaller()
Helper to get the caller (the parent function).
Debug location.
A debug info location.
Definition: DebugLoc.h:33
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
The default (manual heuristics) implementation of the InlineAdvisor.
Capture state between an inlining decision having had been made, and its impact being observable.
Definition: InlineAdvisor.h:75
Function *const Callee
Function *const Caller
Caller and Callee are pre-inlining.
virtual void recordInliningWithCalleeDeletedImpl()
virtual void recordInliningImpl()
const BasicBlock *const Block
void recordInlining()
Exactly one of the record* APIs must be called.
OptimizationRemarkEmitter & ORE
virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result)
InlineAdvisor *const Advisor
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
virtual void recordUnattemptedInliningImpl()
const DebugLoc DLoc
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
static AnalysisKey Key
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
InlineAdvisor(InlineAdvisor &&)=delete
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
Represents the cost of inlining a function.
Definition: InlineCost.h:90
int getThreshold() const
Get the threshold against which the cost was computed.
Definition: InlineCost.h:151
const char * getReason() const
Get the reason of Always or Never.
Definition: InlineCost.h:166
int getCost() const
Get the inline cost estimate.
Definition: InlineCost.h:145
bool isAlways() const
Definition: InlineCost.h:139
int getCostDelta() const
Get the cost delta from the threshold for inlining.
Definition: InlineCost.h:175
bool isNever() const
Definition: InlineCost.h:140
InlineResult is basically true or false.
Definition: InlineCost.h:180
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:563
An SCC of the call graph.
iterator begin() const
A lazily constructed view of the call graph of a module.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:299
The optimization diagnostic interface.
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:688
Used for dynamically registering InlineAdvisors as plugins.
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
Analysis providing profile information.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition: ilist_node.h:32
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:679
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const int LastCallToStaticBonus
Definition: InlineCost.h:50
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:480
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:43
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:76
@ FullLTOPreLink
Full LTO prelink phase.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
@ None
No LTO/ThinLTO behavior needed.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
std::optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
std::optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:293
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: Analysis.h:28
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Used in the streaming interface as the general argument type.
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:59
ThinOrFullLTOPhase LTOPhase
Definition: InlineAdvisor.h:60
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:206
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.
Definition: InlineCost.h:236