32#define DEBUG_TYPE "inline"
33#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
34#define LLVM_HAVE_TF_AOT
40STATISTIC(NumCallerCallersAnalyzed,
"Number of caller-callers analyzed");
46 cl::desc(
"Enable adding inline-remark attribute to"
47 " callsites processed by inliner but decided"
48 " to be not inlined"));
52 cl::desc(
"Enable deferred inlining"));
58 cl::desc(
"Scale to limit the cost of inline deferral"),
63 cl::desc(
"If true, annotate inline advisor remarks "
64 "with LTO and pass information."));
76 bool IsInliningMandatory)
83 if (IsInliningRecommended)
86 Remark <<
": always inline attribute";
91 if (IsInliningRecommended)
94 "NotInlined", DLoc,
Block)
95 <<
"'" <<
NV(
"Callee", Callee) <<
"' is not AlwaysInline into '"
96 <<
NV(
"Caller", Caller)
97 <<
"': " <<
NV(
"Reason", Result.getFailureReason());
102 assert(!IsInliningRecommended &&
"Expected to attempt inlining");
107void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
115 <<
"'" <<
NV(
"Callee",
Callee) <<
"' is not inlined into '"
117 <<
"': " <<
NV(
"Reason",
Result.getFailureReason());
121void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
128void DefaultInlineAdvice::recordInliningImpl() {
140 .getCachedResult<ProfileSummaryAnalysis>(
141 *CB.
getParent()->getParent()->getParent());
156 auto GetInlineCost = [&](
CallBase &CB) {
157 bool RemarksEnabled =
158 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
160 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
161 GetBFI, PSI, RemarksEnabled ? &ORE :
nullptr);
164 CB, CalleeTTI, GetInlineCost, ORE,
168std::unique_ptr<InlineAdvice>
169DefaultInlineAdvisor::getAdviceImpl(
CallBase &CB) {
171 return std::make_unique<DefaultInlineAdvice>(
178 bool IsInliningRecommended)
181 IsInliningRecommended(IsInliningRecommended) {}
183void InlineAdvice::recordInlineStatsIfNeeded() {
190 recordInlineStatsIfNeeded();
196 recordInlineStatsIfNeeded();
209 Advisor.reset(DA.Factory(M,
FAM, Params, IC));
212 auto GetDefaultAdvice = [&
FAM, Params](
CallBase &CB) {
214 return OIC.has_value();
224 std::move(Advisor), ReplaySettings,
229#ifdef LLVM_HAVE_TFLITE
253 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
276 TotalSecondaryCost = 0;
278 int CandidateCost = IC.
getCost() - 1;
281 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
283 bool InliningPreventsSomeOuterInline =
false;
284 unsigned NumCallerUsers = 0;
285 for (
User *U : Caller->users()) {
286 CallBase *CS2 = dyn_cast<CallBase>(U);
292 ApplyLastCallBonus =
false;
297 ++NumCallerCallersAnalyzed;
299 ApplyLastCallBonus =
false;
309 InliningPreventsSomeOuterInline =
true;
310 TotalSecondaryCost += IC2.
getCost();
315 if (!InliningPreventsSomeOuterInline)
322 if (ApplyLastCallBonus)
328 return TotalSecondaryCost < IC.
getCost();
330 int TotalCost = TotalSecondaryCost + IC.
getCost() * NumCallerUsers;
332 return TotalCost < Allowance;
340template <
class RemarkT>
344 R <<
"(cost=always)";
352 R <<
": " <<
ore::NV(
"Reason", Reason);
376std::optional<InlineCost>
389 <<
", Call: " << CB <<
"\n");
395 <<
", Call: " << CB <<
"\n");
399 <<
"'" <<
NV(
"Callee", Callee) <<
"' not inlined into '"
400 <<
NV(
"Caller", Caller)
401 <<
"' because it should never be inlined " << IC;
406 <<
"'" <<
NV(
"Callee", Callee) <<
"' not inlined into '"
407 <<
NV(
"Caller", Caller) <<
"' because too costly to inline "
415 int TotalSecondaryCost = 0;
417 TotalSecondaryCost, GetInlineCost)) {
420 <<
", outer Cost = " << TotalSecondaryCost <<
'\n');
424 <<
"Not inlining. Cost of inlining '" <<
NV(
"Callee", Callee)
425 <<
"' increases the cost of inlining '" <<
NV(
"Caller", Caller)
426 <<
"' in other contexts";
442 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
444 CallSiteLoc <<
" @ ";
449 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
450 uint32_t Discriminator = DIL->getBaseDiscriminator();
451 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
453 Name = DIL->getScope()->getSubprogram()->getName();
454 CallSiteLoc <<
Name.str() <<
":" << llvm::utostr(
Offset);
455 if (
Format.outputColumn())
456 CallSiteLoc <<
":" << llvm::utostr(DIL->getColumn());
457 if (
Format.outputDiscriminator() && Discriminator)
458 CallSiteLoc <<
"." << llvm::utostr(Discriminator);
462 return CallSiteLoc.
str();
471 Remark <<
" at callsite ";
472 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
475 unsigned int Offset = DIL->getLine();
476 Offset -= DIL->getScope()->getSubprogram()->getLine();
477 unsigned int Discriminator = DIL->getBaseDiscriminator();
478 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
480 Name = DIL->getScope()->getSubprogram()->getName();
482 <<
ore::NV(
"Column", DIL->getColumn());
497 StringRef RemarkName = AlwaysInline ?
"AlwaysInline" :
"Inlined";
500 Remark <<
"'" <<
ore::NV(
"Callee", &Callee) <<
"' inlined into '"
501 <<
ore::NV(
"Caller", &Caller) <<
"'";
512 bool ForProfileContext,
const char *
PassName) {
516 if (ForProfileContext)
517 Remark <<
" to match profiling context";
518 Remark <<
" with " << IC;
524 std::optional<InlineContext> IC)
531 std::make_unique<ImportedFunctionsInliningStatistics>();
546 return std::make_unique<MandatoryInlineAdvice>(
this, CB,
getCallerORE(CB),
567 return "always-inline";
569 return "cgscc-inline";
571 return "early-inline";
575 return "module-inline";
577 return "replay-cgscc-inline";
579 return "replay-sample-profile-inline";
581 return "sample-profile-inline";
603 auto TrivialDecision =
606 if (TrivialDecision) {
607 if (TrivialDecision->isSuccess())
616 bool MandatoryOnly) {
633 OS <<
"No Inline Advisor\n";
635 IA->getAdvisor()->print(OS);
642 const auto &MAMProxy =
645 if (InitialC.
size() == 0) {
646 OS <<
"SCC is empty!\n";
649 Module &M = *InitialC.
begin()->getFunction().getParent();
652 OS <<
"No Inline Advisor\n";
654 IA->getAdvisor()->print(OS);
static const Function * getParent(const Value *V)
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
static bool shouldBeDeferred(Function *Caller, TargetTransformInfo &CalleeTTI, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
static std::optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
static const char * getInlineAdvisorContext(InlinePass IP)
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
static const Function * getCalledFunction(const Value *V)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static const char PassName[]
A container for analyses that lazily runs them and caches their results.
bool isPassRegistered() const
Returns true if the specified analysis pass is registered.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
LLVM Basic Block Representation.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
DILocation * get() const
Get the underlying DILocation.
The default (manual heuristics) implementation of the InlineAdvisor.
Capture state between an inlining decision having had been made, and its impact being observable.
Function *const Caller
Caller and Callee are pre-inlining.
virtual void recordInliningWithCalleeDeletedImpl()
virtual void recordInliningImpl()
const BasicBlock *const Block
void recordInlining()
Exactly one of the record* APIs must be called.
OptimizationRemarkEmitter & ORE
virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result)
InlineAdvisor *const Advisor
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
virtual void recordUnattemptedInliningImpl()
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
InlineAdvisor(InlineAdvisor &&)=delete
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
Represents the cost of inlining a function.
int getThreshold() const
Get the threshold against which the cost was computed.
const char * getReason() const
Get the reason of Always or Never.
int getCost() const
Get the inline cost estimate.
int getCostDelta() const
Get the cost delta from the threshold for inlining.
InlineResult is basically true or false.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
A Module instance is used to store all the information related to an LLVM module.
LLVMContext & getContext() const
Get the global data context.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Used for dynamically registering InlineAdvisors as plugins.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Analysis providing profile information.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LLVMContext & getContext() const
All values hold a context through their type.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ ReplaySampleProfileInliner
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
std::optional< InlineCost > shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
@ FullLTOPreLink
Full LTO prelink phase.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
@ None
No LTO/ThinLTO behavior needed.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
std::optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Used in the streaming interface as the general argument type.
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
ThinOrFullLTOPhase LTOPhase
Thresholds to tune inline cost analysis.
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.